33ab928e1b
We currently save the hostname of a proc when we create the ompi_proc_t for it. This was originally done because the only method we had for discovering the host of a proc was to include that info in the modex, and we had to therefore store it somewhere proc-local. Obviously, this ccarried a memory penalty for storing all those strings, and so we added a "cutoff" parameter so that we wouldn't collect hostnames above a certain number of procs. Unfortunately, this still results in an 8-byte/proc memory cost as we have a char* pointer in the opal_proc_t that is contained in the ompi_proc_t so that we can store the hostname of the other procs if we fall below the cutoff. At scale, this can consume a fair amount of memory. With the switch to relying on PMIx, there is no longer a need to cache the proc hostnames. Using the "optional" feature of PMIx_Get, we restrict the retrieval to be purely proc-local - i.e., we retrieve the info either via shared memory or from within the proc-internal hash storage (depending upon the active PMIx components). Thus, the retrieval of a hostname is purely a local operation involving no communication. All RM's are required to provide a complete hostname map of all procs at startup. Thus, we have full access to all hostnames without including them in a modex or having to cache them on each proc. This allows us to remove the char* pointer from the opal_proc_t, saving us 8-bytes/proc. Unfortunately, PMIx_Get does not currently support the return of a static pointer to memory. Thus, even though PMIx has the hostname in its memory, it can only return a malloc'd version of it. I have therefore ensured that the return from opal_get_proc_hostname is consistently malloc'd and free'd wherever used. This shouldn't be a burden as the hostname is only used in one of two circumstances: (a) in an error message (b) in a verbose output for debugging purposes Thus, there should be no performance penalty associated with the malloc/free requirement. PMIx will eventually be returning static pointers, and so we can eventually simplify this method and return a "const char*" - but as noted, this really isn't an issue even today. Signed-off-by: Ralph Castain <rhc@pmix.org>
123 строки
4.8 KiB
C
123 строки
4.8 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* Copyright (c) 2020 Intel, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "opal_config.h"
|
|
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
|
|
#include "opal/mca/mca.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "opal/mca/rcache/rcache.h"
|
|
#include "opal/mca/rcache/base/base.h"
|
|
#include "opal/mca/rcache/base/rcache_base_mem_cb.h"
|
|
#include "rcache_base_vma_tree.h"
|
|
#include "opal/util/show_help.h"
|
|
#include "opal/util/proc.h"
|
|
|
|
#include "opal/runtime/opal_params.h"
|
|
#include "opal/memoryhooks/memory.h"
|
|
|
|
mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void *user_data,
|
|
struct mca_rcache_base_resources_t* resources)
|
|
{
|
|
mca_rcache_base_component_t* component = NULL;
|
|
mca_rcache_base_module_t* module = NULL;
|
|
mca_base_component_list_item_t *cli;
|
|
mca_rcache_base_selected_module_t *sm;
|
|
|
|
/* on the very first creation of a module we init the memory
|
|
callback */
|
|
if (!mca_rcache_base_used_mem_hooks) {
|
|
/* Use the memory hooks if leave_pinned or
|
|
* leave_pinned_pipeline is enabled (note that either of these
|
|
* leave_pinned variables may have been set by a user MCA
|
|
* param or elsewhere in the code base). Yes, we could havexc
|
|
* coded this more succinctly, but this is more clear. Do not
|
|
* check memory hooks if the rcache does not provide an
|
|
* range invalidation function.. */
|
|
if (opal_leave_pinned != 0 || opal_leave_pinned_pipeline) {
|
|
/* open the memory manager components. Memory hooks may be
|
|
triggered before this (any time after mem_free_init(),
|
|
actually). This is a hook available for memory manager hooks
|
|
without good initialization routine support */
|
|
(void) mca_base_framework_open (&opal_memory_base_framework, 0);
|
|
|
|
if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
|
|
((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
|
|
opal_mem_hooks_support_level())) {
|
|
if (-1 == opal_leave_pinned) {
|
|
opal_leave_pinned = !opal_leave_pinned_pipeline;
|
|
}
|
|
opal_mem_hooks_register_release(mca_rcache_base_mem_cb, NULL);
|
|
} else if (1 == opal_leave_pinned || opal_leave_pinned_pipeline) {
|
|
opal_show_help("help-rcache-base.txt", "leave pinned failed",
|
|
true, name, OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
|
|
opal_process_info.nodename);
|
|
return NULL;
|
|
}
|
|
|
|
/* Set this to true so that rcache_base_close knows to
|
|
cleanup */
|
|
mca_rcache_base_used_mem_hooks = 1;
|
|
}
|
|
}
|
|
|
|
OPAL_LIST_FOREACH(cli, &opal_rcache_base_framework.framework_components, mca_base_component_list_item_t) {
|
|
component = (mca_rcache_base_component_t *) cli->cli_component;
|
|
if(0 == strcmp(component->rcache_version.mca_component_name, name)) {
|
|
module = component->rcache_init (resources);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ( NULL == module ) {
|
|
return NULL;
|
|
}
|
|
|
|
sm = OBJ_NEW(mca_rcache_base_selected_module_t);
|
|
sm->rcache_component = component;
|
|
sm->rcache_module = module;
|
|
sm->user_data = user_data;
|
|
opal_list_append(&mca_rcache_base_modules, (opal_list_item_t*) sm);
|
|
|
|
return module;
|
|
}
|
|
|
|
int mca_rcache_base_module_destroy(mca_rcache_base_module_t *module)
|
|
{
|
|
mca_rcache_base_selected_module_t *sm, *next;
|
|
|
|
OPAL_LIST_FOREACH_SAFE(sm, next, &mca_rcache_base_modules, mca_rcache_base_selected_module_t) {
|
|
if (module == sm->rcache_module) {
|
|
opal_list_remove_item(&mca_rcache_base_modules, (opal_list_item_t*)sm);
|
|
if (NULL != sm->rcache_module->rcache_finalize) {
|
|
sm->rcache_module->rcache_finalize(sm->rcache_module);
|
|
}
|
|
OBJ_RELEASE(sm);
|
|
return OPAL_SUCCESS;
|
|
}
|
|
}
|
|
|
|
return OPAL_ERR_NOT_FOUND;
|
|
}
|