1
1

Merge pull request #4704 from ggouaillardet/topic/regx_misc

orte/regx: fix, revamp and enhancement
Этот коммит содержится в:
Ralph Castain 2018-01-12 06:50:58 -08:00 коммит произвёл GitHub
родитель 1821c6530c c988011afd
Коммит e35347f9e3
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 1870 добавлений и 1305 удалений

Просмотреть файл

@ -1,5 +1,7 @@
#
# Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
# Copyright (c) 2018 Research Organization for Information Science
# and Technology (RIST). All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -11,5 +13,6 @@ headers += \
base/base.h
libmca_regx_la_SOURCES += \
base/regx_base_default_fns.c \
base/regx_base_frame.c \
base/regx_base_select.c

Просмотреть файл

@ -35,6 +35,40 @@ ORTE_DECLSPEC extern mca_base_framework_t orte_regx_base_framework;
/* select all components */
ORTE_DECLSPEC int orte_regx_base_select(void);
/*
* common stuff
*/
typedef struct {
opal_list_item_t super;
int vpid;
int cnt;
int slots;
orte_topology_t *t;
} orte_regex_range_t;
OBJ_CLASS_DECLARATION(orte_regex_range_t);
typedef struct {
/* list object */
opal_list_item_t super;
char *prefix;
char *suffix;
int num_digits;
opal_list_t ranges;
} orte_regex_node_t;
END_C_DECLS
OBJ_CLASS_DECLARATION(orte_regex_node_t);
ORTE_DECLSPEC extern int orte_regx_base_nidmap_parse(char *regex);
ORTE_DECLSPEC extern int orte_regx_base_encode_nodemap(opal_buffer_t *buffer);
ORTE_DECLSPEC int orte_regx_base_decode_daemon_nodemap(opal_buffer_t *buffer);
ORTE_DECLSPEC int orte_regx_base_generate_ppn(orte_job_t *jdata, char **ppn);
ORTE_DECLSPEC int orte_regx_base_parse_ppn(orte_job_t *jdata, char *regex);
ORTE_DECLSPEC int orte_regx_base_extract_node_names(char *regexp, char ***names);
#endif

1281
orte/mca/regx/base/regx_base_default_fns.c Обычный файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -3,6 +3,8 @@
* Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -59,6 +61,7 @@ typedef int (*orte_regx_base_module_init_fn_t)(void);
typedef int (*orte_regx_base_module_nidmap_create_fn_t)(opal_pointer_array_t *pool, char **regex);
typedef int (*orte_regx_base_module_nidmap_parse_fn_t)(char *regex);
typedef int (*orte_regx_base_module_extract_node_names_fn_t)(char *regexp, char ***names);
/* create a regular expression describing the nodes in the
* allocation */
@ -87,6 +90,7 @@ typedef struct {
orte_regx_base_module_init_fn_t init;
orte_regx_base_module_nidmap_create_fn_t nidmap_create;
orte_regx_base_module_nidmap_parse_fn_t nidmap_parse;
orte_regx_base_module_extract_node_names_fn_t extract_node_names;
orte_regx_base_module_encode_nodemap_fn_t encode_nodemap;
orte_regx_base_module_decode_daemon_nodemap_fn_t decode_daemon_nodemap;
orte_regx_base_module_build_daemon_nidmap_fn_t build_daemon_nidmap;

Просмотреть файл

@ -1,5 +1,8 @@
/*
* Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
* Copyright (c) 2018 IBM Corporation. All rights reserved.
* Copyright (c) 2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -23,52 +26,306 @@
#include "orte/runtime/orte_globals.h"
#include "orte/util/name_fns.h"
#include "orte/util/show_help.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/mca/rmaps/base/base.h"
#include "orte/mca/routed/routed.h"
#include "orte/mca/regx/base/base.h"
#include "regx_reverse.h"
static int nidmap_create(opal_pointer_array_t *pool, char **regex);
static int nidmap_parse(char *regex);
static int encode_nodemap(opal_buffer_t *buffer);
static int decode_daemon_nodemap(opal_buffer_t *buffer);
static int generate_ppn(orte_job_t *jdata, char **ppn);
static int parse_ppn(orte_job_t *jdata, char *ppn);
orte_regx_base_module_t orte_regx_reverse_module = {
.nidmap_create = nidmap_create,
.nidmap_parse = nidmap_parse,
.encode_nodemap = encode_nodemap,
.decode_daemon_nodemap = decode_daemon_nodemap,
.generate_ppn = generate_ppn,
.parse_ppn = parse_ppn
.nidmap_parse = orte_regx_base_nidmap_parse,
.extract_node_names = orte_regx_base_extract_node_names,
.encode_nodemap = orte_regx_base_encode_nodemap,
.decode_daemon_nodemap = orte_regx_base_decode_daemon_nodemap,
.generate_ppn = orte_regx_base_generate_ppn,
.parse_ppn = orte_regx_base_parse_ppn
};
static int nidmap_create(opal_pointer_array_t *pool, char **regex)
{
return ORTE_ERR_NOT_IMPLEMENTED;
}
char *node;
char prefix[ORTE_MAX_NODE_PREFIX];
int i, j, n, len, startnum, nodenum, numdigits;
bool found;
char *suffix, *sfx, *nodenames;
orte_regex_node_t *ndreg;
orte_regex_range_t *range, *rng;
opal_list_t nodenms, dvpids;
opal_list_item_t *item, *itm2;
char **regexargs = NULL, *tmp, *tmp2;
orte_node_t *nptr;
orte_vpid_t vpid;
static int nidmap_parse(char *regex)
{
return ORTE_ERR_NOT_IMPLEMENTED;
}
OBJ_CONSTRUCT(&nodenms, opal_list_t);
OBJ_CONSTRUCT(&dvpids, opal_list_t);
static int encode_nodemap(opal_buffer_t *buffer)
{
return ORTE_ERR_NOT_IMPLEMENTED;
}
rng = NULL;
for (n=0; n < pool->size; n++) {
if (NULL == (nptr = (orte_node_t*)opal_pointer_array_get_item(pool, n))) {
continue;
}
/* if no daemon has been assigned, then this node is not being used */
if (NULL == nptr->daemon) {
vpid = -1; // indicates no daemon assigned
} else {
vpid = nptr->daemon->name.vpid;
}
/* deal with the daemon vpid - see if it is next in the
* current range */
if (NULL == rng) {
/* just starting */
rng = OBJ_NEW(orte_regex_range_t);
rng->vpid = vpid;
rng->cnt = 1;
opal_list_append(&dvpids, &rng->super);
} else if (UINT32_MAX == vpid) {
if (-1 == rng->vpid) {
rng->cnt++;
} else {
/* need to start another range */
rng = OBJ_NEW(orte_regex_range_t);
rng->vpid = vpid;
rng->cnt = 1;
opal_list_append(&dvpids, &rng->super);
}
} else if (-1 == rng->vpid) {
/* need to start another range */
rng = OBJ_NEW(orte_regex_range_t);
rng->vpid = vpid;
rng->cnt = 1;
opal_list_append(&dvpids, &rng->super);
} else {
/* is this the next in line */
if (vpid == (orte_vpid_t)(rng->vpid + rng->cnt)) {
rng->cnt++;
} else {
/* need to start another range */
rng = OBJ_NEW(orte_regex_range_t);
rng->vpid = vpid;
rng->cnt = 1;
opal_list_append(&dvpids, &rng->super);
}
}
node = nptr->name;
opal_output_verbose(5, orte_regx_base_framework.framework_output,
"%s PROCESS NODE <%s>",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
node);
/* determine this node's prefix by looking for first digit char */
len = strlen(node);
startnum = -1;
memset(prefix, 0, ORTE_MAX_NODE_PREFIX);
numdigits = 0;
static int decode_daemon_nodemap(opal_buffer_t *buffer)
{
return ORTE_ERR_NOT_IMPLEMENTED;
}
/* Valid hostname characters are:
* - ascii letters, digits, and the '-' character.
* Determine the prefix in reverse to better support hostnames like:
* c712f6n01, c699c086 where there are sets of digits, and the lowest
* set changes most frequently.
*/
startnum = -1;
memset(prefix, 0, ORTE_MAX_NODE_PREFIX);
numdigits = 0;
for (i=len-1; i >= 0; i--) {
// Count all of the digits
if( isdigit(node[i]) ) {
numdigits++;
continue;
}
else {
// At this point everything at and above position 'i' is prefix.
for( j = 0; j <= i; ++j) {
prefix[j] = node[j];
}
startnum = j;
break;
}
}
static int generate_ppn(orte_job_t *jdata, char **ppn)
{
return ORTE_ERR_NOT_IMPLEMENTED;
}
opal_output_verbose(5, orte_regx_base_framework.framework_output,
"%s PROCESS NODE <%s> : reverse / prefix \"%s\" / numdigits %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
node, prefix, numdigits);
static int parse_ppn(orte_job_t *jdata, char *ppn)
{
return ORTE_ERR_NOT_IMPLEMENTED;
if (startnum < 0) {
/* can't compress this name - just add it to the list */
ndreg = OBJ_NEW(orte_regex_node_t);
ndreg->prefix = strdup(node);
opal_list_append(&nodenms, &ndreg->super);
continue;
}
/* convert the digits and get any suffix */
nodenum = strtol(&node[startnum], &sfx, 10);
if (NULL != sfx) {
suffix = strdup(sfx);
} else {
suffix = NULL;
}
/* is this node name already on our list? */
found = false;
for (item = opal_list_get_first(&nodenms);
!found && item != opal_list_get_end(&nodenms);
item = opal_list_get_next(item)) {
ndreg = (orte_regex_node_t*)item;
if (0 < strlen(prefix) && NULL == ndreg->prefix) {
continue;
}
if (0 == strlen(prefix) && NULL != ndreg->prefix) {
continue;
}
if (0 < strlen(prefix) && NULL != ndreg->prefix
&& 0 != strcmp(prefix, ndreg->prefix)) {
continue;
}
if (NULL == suffix && NULL != ndreg->suffix) {
continue;
}
if (NULL != suffix && NULL == ndreg->suffix) {
continue;
}
if (NULL != suffix && NULL != ndreg->suffix &&
0 != strcmp(suffix, ndreg->suffix)) {
continue;
}
if (numdigits != ndreg->num_digits) {
continue;
}
/* found a match - flag it */
found = true;
/* get the last range on this nodeid - we do this
* to preserve order
*/
range = (orte_regex_range_t*)opal_list_get_last(&ndreg->ranges);
if (NULL == range) {
/* first range for this nodeid */
range = OBJ_NEW(orte_regex_range_t);
range->vpid = nodenum;
range->cnt = 1;
opal_list_append(&ndreg->ranges, &range->super);
break;
}
/* see if the node number is out of sequence */
if (nodenum != (range->vpid + range->cnt)) {
/* start a new range */
range = OBJ_NEW(orte_regex_range_t);
range->vpid = nodenum;
range->cnt = 1;
opal_list_append(&ndreg->ranges, &range->super);
break;
}
/* everything matches - just increment the cnt */
range->cnt++;
break;
}
if (!found) {
/* need to add it */
ndreg = OBJ_NEW(orte_regex_node_t);
if (0 < strlen(prefix)) {
ndreg->prefix = strdup(prefix);
}
if (NULL != suffix) {
ndreg->suffix = strdup(suffix);
}
ndreg->num_digits = numdigits;
opal_list_append(&nodenms, &ndreg->super);
/* record the first range for this nodeid - we took
* care of names we can't compress above
*/
range = OBJ_NEW(orte_regex_range_t);
range->vpid = nodenum;
range->cnt = 1;
opal_list_append(&ndreg->ranges, &range->super);
}
if (NULL != suffix) {
free(suffix);
}
}
/* begin constructing the regular expression */
while (NULL != (item = opal_list_remove_first(&nodenms))) {
ndreg = (orte_regex_node_t*)item;
/* if no ranges, then just add the name */
if (0 == opal_list_get_size(&ndreg->ranges)) {
if (NULL != ndreg->prefix) {
/* solitary node */
asprintf(&tmp, "%s", ndreg->prefix);
opal_argv_append_nosize(&regexargs, tmp);
free(tmp);
}
OBJ_RELEASE(ndreg);
continue;
}
/* start the regex for this nodeid with the prefix */
if (NULL != ndreg->prefix) {
asprintf(&tmp, "%s[%d:", ndreg->prefix, ndreg->num_digits);
} else {
asprintf(&tmp, "[%d:", ndreg->num_digits);
}
/* add the ranges */
while (NULL != (itm2 = opal_list_remove_first(&ndreg->ranges))) {
range = (orte_regex_range_t*)itm2;
if (1 == range->cnt) {
asprintf(&tmp2, "%s%u,", tmp, range->vpid);
} else {
asprintf(&tmp2, "%s%u-%u,", tmp, range->vpid, range->vpid + range->cnt - 1);
}
free(tmp);
tmp = tmp2;
OBJ_RELEASE(range);
}
/* replace the final comma */
tmp[strlen(tmp)-1] = ']';
if (NULL != ndreg->suffix) {
/* add in the suffix, if provided */
asprintf(&tmp2, "%s%s", tmp, ndreg->suffix);
free(tmp);
tmp = tmp2;
}
opal_argv_append_nosize(&regexargs, tmp);
free(tmp);
OBJ_RELEASE(ndreg);
}
/* assemble final result */
nodenames = opal_argv_join(regexargs, ',');
/* cleanup */
opal_argv_free(regexargs);
OBJ_DESTRUCT(&nodenms);
/* do the same for the vpids */
tmp = NULL;
while (NULL != (item = opal_list_remove_first(&dvpids))) {
rng = (orte_regex_range_t*)item;
if (1 < rng->cnt) {
if (NULL == tmp) {
asprintf(&tmp, "%u(%u)", rng->vpid, rng->cnt);
} else {
asprintf(&tmp2, "%s,%u(%u)", tmp, rng->vpid, rng->cnt);
free(tmp);
tmp = tmp2;
}
} else {
if (NULL == tmp) {
asprintf(&tmp, "%u", rng->vpid);
} else {
asprintf(&tmp2, "%s,%u", tmp, rng->vpid);
free(tmp);
tmp = tmp2;
}
}
OBJ_RELEASE(rng);
}
OPAL_LIST_DESTRUCT(&dvpids);
/* now concatenate the results into one string */
asprintf(&tmp2, "%s@%s", nodenames, tmp);
free(nodenames);
free(tmp);
*regex = tmp2;
return ORTE_SUCCESS;
}

Просмотреть файл

@ -12,6 +12,8 @@
# Copyright (c) 2012 Los Alamos National Security, LLC. All rights
# reserved.
# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2018 Research Organization for Information Science
# and Technology (RIST). All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -30,7 +32,8 @@ AM_CPPFLAGS = -I$(top_srcdir)/test/support
# opal_os_create_dirpath \
# opal_argv \
# opal_basename \
# opal_path_nfs
# opal_path_nfs \
# orte_nidmap
check_PROGRAMS = \
@ -119,6 +122,12 @@ opal_path_nfs_DEPENDENCIES = $(opal_path_nfs_LDADD)
# $(top_builddir)/test/support/libsupport.a
#orte_universe_setup_file_io_DEPENDENCIES = $(orte_universe_setup_file_io_LDADD)
#orte_nidmap_SOURCES = orte_nidmap.c
#orte_nidmap_LDADD = \
# $(top_builddir)/orte/libopen-rte.la \
# $(top_builddir)/test/support/libsupport.a
#orte_nidmap_DEPENDENCIES = $(orte_nidmap_LDADD)
bipartite_graph_SOURCES = bipartite_graph.c
bipartite_graph_LDADD = \
$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \

240
test/util/orte_nidmap.c Обычный файл
Просмотреть файл

@ -0,0 +1,240 @@
/*
* Copyright (c) 2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "orte_config.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif /* HAVE_UNISTD_H */
#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif /* HAVE_SYS_PARAM_H */
#include <sys/stat.h>
#include "support.h"
#include "opal/runtime/opal.h"
#include "orte/include/orte/frameworks.h"
#include "orte/constants.h"
#include "orte/runtime/orte_globals.h"
#include "orte/mca/regx/regx.h"
#include "orte/mca/regx/base/base.h"
static void check (const char *value, const char * expected) {
size_t sz = strlen(expected);
assert(strlen(value) >= sz);
assert(0 == strncmp(value, expected, sz-1));
assert('@' == value[sz] || '\0' == value[sz]);
}
int main(int argc, char* argv[])
{
char * regex;
char ** nodes;
opal_pointer_array_t pool;
orte_node_t * node;
orte_proc_info(); /* initialize proc info structure */
test_init("orte_nidmap");
opal_init(&argc, &argv);
orte_init(&argc, &argv, ORTE_PROC_TYPE_NONE);
if (ORTE_SUCCESS != mca_base_framework_open(&orte_regx_base_framework, 0)) {
return -1;
}
if (ORTE_SUCCESS != orte_regx_base_select()) {
return -1;
}
OBJ_CONSTRUCT(&pool, opal_pointer_array_t);
orte_node_pool = OBJ_NEW(opal_pointer_array_t);
orte_job_data = OBJ_NEW(opal_hash_table_t);
orte_job_t *jdata = OBJ_NEW(orte_job_t);
jdata->jobid = 1;
orte_process_info.my_name.jobid = 1;
opal_hash_table_init(orte_job_data, 1);
opal_hash_table_set_value_uint32(orte_job_data, jdata->jobid, jdata);
node = OBJ_NEW(orte_node_t);
node->daemon = OBJ_NEW(orte_proc_t);
node->daemon->name.jobid = 1;
node->daemon->name.vpid = 0;
node->name = "n0";
opal_pointer_array_add(&pool, node);
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n0 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n0");
assert(NULL == nodes[1]);
node = OBJ_NEW(orte_node_t);
node->daemon = OBJ_NEW(orte_proc_t);
node->daemon->name.vpid = 1;
node->name = "n1";
opal_pointer_array_add(&pool, node);
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n0,n1 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n0");
check(nodes[1], "n1");
assert(NULL == nodes[2]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "n-0";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "n-1";
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n-0,n-1 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n-0");
check(nodes[1], "n-1");
assert(NULL == nodes[2]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "n-000";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "n-001";
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n-000,n-001 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n-000");
check(nodes[1], "n-001");
assert(NULL == nodes[2]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "n9";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "n10";
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n9,n10 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n9");
check(nodes[1], "n10");
assert(NULL == nodes[2]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "n99";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "n100";
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n99,n100 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n99");
check(nodes[1], "n100");
assert(NULL == nodes[2]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "c712f6n01";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "c712f6n02";
node = OBJ_NEW(orte_node_t);
node->daemon = OBJ_NEW(orte_proc_t);
node->daemon->name.vpid = 2;
node->name = "c712f6n03";
opal_pointer_array_add(&pool, node);
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for c712f6n01,c712f6n02,c712f6n03 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "c712f6n01");
check(nodes[1], "c712f6n02");
check(nodes[2], "c712f6n03");
assert(NULL == nodes[3]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "n01c712";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "n02c712";
node = opal_pointer_array_get_item(&pool, 2);
node->name = "n03c712";
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for n01c712,n02c712,n03c712 is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "n01c712");
check(nodes[1], "n02c712");
check(nodes[2], "n03c712");
assert(NULL == nodes[3]);
node = opal_pointer_array_get_item(&pool, 0);
node->name = "c8n";
node = opal_pointer_array_get_item(&pool, 1);
node->name = "c9n";
node = opal_pointer_array_get_item(&pool, 2);
node->name = "c10n";
regex = NULL;
orte_regx.nidmap_create(&pool, &regex);
printf ("regex for c8n,c9n,c10n is %s\n", regex);
nodes = NULL;
orte_regx.extract_node_names(regex, &nodes);
check(nodes[0], "c8n");
check(nodes[1], "c9n");
check(nodes[2], "c10n");
assert(NULL == nodes[3]);
/* clean up */
orte_proc_info_finalize();
test_finalize();
return 0;
}