From be3ef777392347aa4560fb4eaa13075d2e77ed6e Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Sat, 21 Jan 2017 14:03:19 -0800 Subject: [PATCH] Improve packing efficiency by raising the initial buffer size and modifying the extension code. Flag if a job map has had its nodes added so we don't have to loop repeatedly to check it. Signed-off-by: Ralph Castain --- opal/dss/dss_internal.h | 6 ++-- opal/dss/dss_internal_functions.c | 33 ++++++------------- orte/mca/odls/base/odls_base_default_fns.c | 21 +++++------- orte/mca/rmaps/base/rmaps_base_map_job.c | 2 ++ orte/orted/pmix/pmix_server_register_fns.c | 5 +++ .../data_type_support/orte_dt_unpacking_fns.c | 8 +++-- orte/util/attr.h | 4 +-- 7 files changed, 36 insertions(+), 43 deletions(-) diff --git a/opal/dss/dss_internal.h b/opal/dss/dss_internal.h index 2c1e3af73b..a2514379ce 100644 --- a/opal/dss/dss_internal.h +++ b/opal/dss/dss_internal.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. @@ -47,12 +47,12 @@ BEGIN_C_DECLS /* * The default starting chunk size */ -#define OPAL_DSS_DEFAULT_INITIAL_SIZE 128 +#define OPAL_DSS_DEFAULT_INITIAL_SIZE 2048 /* * The default threshold size when we switch from doubling the * buffer size to addatively increasing it */ -#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 1024 +#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 4096 /* * Internal type corresponding to size_t. Do not use this in diff --git a/opal/dss/dss_internal_functions.c b/opal/dss/dss_internal_functions.c index 09fef89259..3c8b62c663 100644 --- a/opal/dss/dss_internal_functions.c +++ b/opal/dss/dss_internal_functions.c @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,6 +36,7 @@ char* opal_dss_buffer_extend(opal_buffer_t *buffer, size_t bytes_to_add) { size_t required, to_alloc; size_t pack_offset, unpack_offset; + char *tmp; /* Check to see if we have enough space already */ @@ -43,34 +45,19 @@ char* opal_dss_buffer_extend(opal_buffer_t *buffer, size_t bytes_to_add) } required = buffer->bytes_used + bytes_to_add; - if(required >= (size_t)opal_dss_threshold_size) { - to_alloc = ((required + opal_dss_threshold_size - 1) - / opal_dss_threshold_size) * opal_dss_threshold_size; + if (required >= (size_t)opal_dss_threshold_size) { + to_alloc = (required + opal_dss_threshold_size - 1) & ~(opal_dss_threshold_size - 1); } else { - to_alloc = buffer->bytes_allocated; - if(0 == to_alloc) { - to_alloc = opal_dss_initial_size; - } - while(to_alloc < required) { - to_alloc <<= 1; - } + to_alloc = buffer->bytes_allocated ? buffer->bytes_allocated : (size_t)opal_dss_initial_size; } - if (NULL != buffer->base_ptr) { - pack_offset = ((char*) buffer->pack_ptr) - ((char*) buffer->base_ptr); - unpack_offset = ((char*) buffer->unpack_ptr) - - ((char*) buffer->base_ptr); - buffer->base_ptr = (char*)realloc(buffer->base_ptr, to_alloc); - } else { - pack_offset = 0; - unpack_offset = 0; - buffer->bytes_used = 0; - buffer->base_ptr = (char*)malloc(to_alloc); - } - - if (NULL == buffer->base_ptr) { + pack_offset = ((char*) buffer->pack_ptr) - ((char*) buffer->base_ptr); + unpack_offset = ((char*) buffer->unpack_ptr) - ((char*) buffer->base_ptr); + tmp = (char*)realloc(buffer->base_ptr, to_alloc); + if (NULL == tmp) { return NULL; } + buffer->base_ptr = tmp; buffer->pack_ptr = ((char*) buffer->base_ptr) + pack_offset; buffer->unpack_ptr = ((char*) buffer->base_ptr) + unpack_offset; buffer->bytes_allocated = to_alloc; diff --git a/orte/mca/odls/base/odls_base_default_fns.c b/orte/mca/odls/base/odls_base_default_fns.c index 57b6687850..3f94acacb1 100644 --- a/orte/mca/odls/base/odls_base_default_fns.c +++ b/orte/mca/odls/base/odls_base_default_fns.c @@ -261,7 +261,6 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer, orte_proc_t *pptr, *dmn; opal_buffer_t *bptr; orte_app_context_t *app; - bool found; orte_node_t *node; bool newmap = false; @@ -409,6 +408,13 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer, if (NULL == jdata->map) { jdata->map = OBJ_NEW(orte_job_map_t); newmap = true; + } else if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED)) { + /* zero all the node map flags */ + for (n=0; n < jdata->map->nodes->size; n++) { + if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, n))) { + ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED); + } + } } /* if we have a file map, then we need to load it */ @@ -454,17 +460,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer, opal_pointer_array_add(dmn->node->procs, pptr); /* add the node to the map, if not already there */ - found = false; - for (k=0; k < jdata->map->nodes->size; k++) { - if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, k))) { - continue; - } - if (node->daemon == dmn) { - found = true; - break; - } - } - if (!found) { + if (!ORTE_FLAG_TEST(dmn->node, ORTE_NODE_FLAG_MAPPED)) { OBJ_RETAIN(dmn->node); opal_pointer_array_add(jdata->map->nodes, dmn->node); if (newmap) { @@ -497,6 +493,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer, app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx); ORTE_FLAG_SET(app, ORTE_APP_FLAG_USED_ON_NODE); } + ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED); } COMPLETE: diff --git a/orte/mca/rmaps/base/rmaps_base_map_job.c b/orte/mca/rmaps/base/rmaps_base_map_job.c index b7cfcd7cc7..072119ab88 100644 --- a/orte/mca/rmaps/base/rmaps_base_map_job.c +++ b/orte/mca/rmaps/base/rmaps_base_map_job.c @@ -389,6 +389,8 @@ void orte_rmaps_base_map_job(int fd, short args, void *cbdata) OBJ_RELEASE(caddy); return; } + /* mark that nodes were assigned to this job */ + ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED); /* if any node is oversubscribed, then check to see if a binding * directive was given - if not, then we want to clear the default diff --git a/orte/orted/pmix/pmix_server_register_fns.c b/orte/orted/pmix/pmix_server_register_fns.c index 22c1a76856..cedb4b4e66 100644 --- a/orte/orted/pmix/pmix_server_register_fns.c +++ b/orte/orted/pmix/pmix_server_register_fns.c @@ -74,6 +74,11 @@ int orte_pmix_server_register_nspace(orte_job_t *jdata) ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid)); + /* if this job has no local procs, then no need to register them */ + if (0 == jdata->num_local_procs) { + return ORTE_SUCCESS; + } + /* setup the info list */ info = OBJ_NEW(opal_list_t); uid = geteuid(); diff --git a/orte/runtime/data_type_support/orte_dt_unpacking_fns.c b/orte/runtime/data_type_support/orte_dt_unpacking_fns.c index 8cf990e95c..226d54d4aa 100644 --- a/orte/runtime/data_type_support/orte_dt_unpacking_fns.c +++ b/orte/runtime/data_type_support/orte_dt_unpacking_fns.c @@ -12,7 +12,7 @@ * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -168,9 +168,9 @@ int orte_dt_unpack_job(opal_buffer_t *buffer, void *dest, return rc; } - /* if the map is NULL, then we din't pack it as there was + /* if the map is NULL, then we didn't pack it as there was * nothing to pack. Instead, we packed a flag to indicate whether or not - * the map is included */ + * the map is included */ n = 1; if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer, &j, &n, ORTE_STD_CNTR))) { @@ -204,6 +204,8 @@ int orte_dt_unpack_job(opal_buffer_t *buffer, void *dest, ORTE_ERROR_LOG(rc); return rc; } + /* mark the map as uninitialized as we don't pack the node map */ + ORTE_FLAG_UNSET(jobs[i], ORTE_JOB_FLAG_MAP_INITIALIZED); /* unpack the attributes */ n=1; diff --git a/orte/util/attr.h b/orte/util/attr.h index d06d76a727..7bad62c5ef 100644 --- a/orte/util/attr.h +++ b/orte/util/attr.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved + * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -89,7 +89,7 @@ typedef uint16_t orte_job_flags_t; #define ORTE_JOB_FLAG_RESTART 0x0200 // #define ORTE_JOB_FLAG_PROCS_MIGRATING 0x0400 // some procs in job are migrating from one node to another #define ORTE_JOB_FLAG_OVERSUBSCRIBED 0x0800 // at least one node in the job is oversubscribed - +#define ORTE_JOB_FLAG_MAP_INITIALIZED 0x1000 // nodes have been assigned to this job map /*** JOB ATTRIBUTE KEYS ***/ #define ORTE_JOB_START_KEY ORTE_NODE_MAX_KEY