diff --git a/ompi/errhandler/errhandler.c b/ompi/errhandler/errhandler.c index 5c832cc597..8ce4c38342 100644 --- a/ompi/errhandler/errhandler.c +++ b/ompi/errhandler/errhandler.c @@ -14,7 +14,7 @@ * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -42,7 +42,7 @@ opal_pointer_array_t ompi_errhandler_f_to_c_table = {{0}}; /* * default errhandler id */ -static int default_errhandler_id = -1; +static size_t default_errhandler_id = SIZE_MAX; /* * Class information @@ -163,7 +163,7 @@ int ompi_errhandler_finalize(void) /* JMS Add stuff here checking for unreleased errorhandlers, similar to communicators, info handles, etc. */ - opal_pmix.deregister_errhandler(default_errhandler_id, NULL, NULL); + opal_pmix.deregister_evhandler(default_errhandler_id, NULL, NULL); /* Remove errhandler F2C table */ @@ -222,7 +222,7 @@ ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type, /* registration callback */ void ompi_errhandler_registration_callback(int status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata) { ompi_errhandler_errtrk_t *errtrk = (ompi_errhandler_errtrk_t*)cbdata; @@ -236,14 +236,15 @@ void ompi_errhandler_registration_callback(int status, * Default errhandler callback */ void ompi_errhandler_callback(int status, - opal_list_t *procs, - opal_list_t *info, - opal_pmix_release_cbfunc_t cbfunc, + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, void *cbdata) { - /* allow the caller to release its data */ + /* tell the event chain engine to go no further - we + * will handle this */ if (NULL != cbfunc) { - cbfunc(cbdata); + cbfunc(OMPI_ERR_HANDLERS_COMPLETE, NULL, NULL, NULL, cbdata); } /* our default action is to abort */ ompi_mpi_abort(MPI_COMM_WORLD, status); diff --git a/ompi/errhandler/errhandler.h b/ompi/errhandler/errhandler.h index 0063b6541b..275acff5f8 100644 --- a/ompi/errhandler/errhandler.h +++ b/ompi/errhandler/errhandler.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2008-2009 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -380,13 +380,13 @@ typedef struct { } ompi_errhandler_errtrk_t; OMPI_DECLSPEC void ompi_errhandler_callback(int status, - opal_list_t *procs, - opal_list_t *info, - opal_pmix_release_cbfunc_t cbfunc, + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, void *cbdata); OMPI_DECLSPEC void ompi_errhandler_registration_callback(int status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata); /** * Check to see if an errhandler is intrinsic. diff --git a/ompi/include/ompi/constants.h b/ompi/include/ompi/constants.h index dbe9001bac..ed9e463853 100644 --- a/ompi/include/ompi/constants.h +++ b/ompi/include/ompi/constants.h @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -63,6 +64,7 @@ enum { OMPI_ERR_BUFFER = OPAL_ERR_BUFFER, OMPI_ERR_SILENT = OPAL_ERR_SILENT, + OMPI_ERR_HANDLERS_COMPLETE = OPAL_ERR_HANDLERS_COMPLETE, OMPI_ERR_REQUEST = OMPI_ERR_BASE - 1, OMPI_ERR_RMA_SYNC = OMPI_ERR_BASE - 2, diff --git a/ompi/mca/rte/orte/rte_orte_module.c b/ompi/mca/rte/orte/rte_orte_module.c index ce35edfda4..cc4e895e4b 100644 --- a/ompi/mca/rte/orte/rte_orte_module.c +++ b/ompi/mca/rte/orte/rte_orte_module.c @@ -95,6 +95,34 @@ void ompi_rte_abort(int error_code, char *fmt, ...) exit(-1); } +static size_t handler = SIZE_MAX; +static bool debugger_register_active = true; +static bool debugger_event_active = true; + +static void _release_fn(int status, + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, + void *cbdata) +{ + /* must let the notifier know we are done */ + if (NULL != cbfunc) { + cbfunc(ORTE_SUCCESS, NULL, NULL, NULL, cbdata); + } + debugger_event_active = false; +} + +static void _register_fn(int status, + size_t evhandler_ref, + void *cbdata) +{ + opal_list_t *codes = (opal_list_t*)cbdata; + + handler = evhandler_ref; + OPAL_LIST_RELEASE(codes); + debugger_register_active = false; +} + /* * Wait for a debugger if asked. We support two ways of waiting for * attaching debuggers -- see big comment in @@ -103,7 +131,8 @@ void ompi_rte_abort(int error_code, char *fmt, ...) void ompi_rte_wait_for_debugger(void) { int debugger; - orte_rml_recv_cb_t xfer; + opal_list_t *codes; + opal_value_t *kv; /* See lengthy comment in orte/tools/orterun/debuggers.c about orte_in_parallel_debugger */ @@ -133,23 +162,23 @@ void ompi_rte_wait_for_debugger(void) #endif } } else { - /* only the rank=0 proc waits for either a message from the - * HNP or for the debugger to attach - everyone else will just - * spin in * the grpcomm barrier in ompi_mpi_init until rank=0 - * joins them. - */ - if (0 != ORTE_PROC_MY_NAME->vpid) { - return; - } - /* VPID 0 waits for a message from the HNP */ - OBJ_CONSTRUCT(&xfer, orte_rml_recv_cb_t); - xfer.active = true; - orte_rml.recv_buffer_nb(OMPI_NAME_WILDCARD, - ORTE_RML_TAG_DEBUGGER_RELEASE, - ORTE_RML_NON_PERSISTENT, - orte_rml_recv_callback, &xfer); - /* let the MPI progress engine run while we wait */ - OMPI_WAIT_FOR_COMPLETION(xfer.active); + /* register an event handler for the ORTE_ERR_DEBUGGER_RELEASE event */ + codes = OBJ_NEW(opal_list_t); + kv = OBJ_NEW(opal_value_t); + kv->key = strdup("errorcode"); + kv->type = OPAL_INT; + kv->data.integer = ORTE_ERR_DEBUGGER_RELEASE; + opal_list_append(codes, &kv->super); + + opal_pmix.register_evhandler(codes, NULL, _release_fn, _register_fn, codes); + /* let the MPI progress engine run while we wait for registration to complete */ + OMPI_WAIT_FOR_COMPLETION(debugger_register_active); + + /* let the MPI progress engine run while we wait for debugger release */ + OMPI_WAIT_FOR_COMPLETION(debugger_event_active); + + /* deregister the event handler */ + opal_pmix.deregister_evhandler(handler, NULL, NULL); } } diff --git a/ompi/mpi/c/lookup_name.c b/ompi/mpi/c/lookup_name.c index 190afd650d..42a71c367c 100644 --- a/ompi/mpi/c/lookup_name.c +++ b/ompi/mpi/c/lookup_name.c @@ -12,10 +12,10 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -94,13 +94,13 @@ int MPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name) rng = OBJ_NEW(opal_value_t); rng->key = strdup(OPAL_PMIX_RANGE); rng->type = OPAL_INT; - rng->data.integer = OPAL_PMIX_NAMESPACE; // share only with procs in same nspace + rng->data.integer = OPAL_PMIX_RANGE_NAMESPACE; // share only with procs in same nspace opal_list_append(&pinfo, &rng->super); } else if (0 == strcmp(range, "session")) { rng = OBJ_NEW(opal_value_t); rng->key = strdup(OPAL_PMIX_RANGE); rng->type = OPAL_INT; - rng->data.integer = OPAL_PMIX_SESSION; // share only with procs in same session + rng->data.integer = OPAL_PMIX_RANGE_SESSION; // share only with procs in same session opal_list_append(&pinfo, &rng->super); } else { /* unrecognized scope */ diff --git a/ompi/mpi/c/publish_name.c b/ompi/mpi/c/publish_name.c index 60ab0da620..24270cea8f 100644 --- a/ompi/mpi/c/publish_name.c +++ b/ompi/mpi/c/publish_name.c @@ -12,10 +12,10 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -94,13 +94,13 @@ int MPI_Publish_name(const char *service_name, MPI_Info info, rng = OBJ_NEW(opal_value_t); rng->key = strdup(OPAL_PMIX_RANGE); rng->type = OPAL_INT; - rng->data.integer = OPAL_PMIX_NAMESPACE; // share only with procs in same nspace + rng->data.integer = OPAL_PMIX_RANGE_NAMESPACE; // share only with procs in same nspace opal_list_append(&values, &rng->super); } else if (0 == strcmp(range, "session")) { rng = OBJ_NEW(opal_value_t); rng->key = strdup(OPAL_PMIX_RANGE); rng->type = OPAL_INT; - rng->data.integer = OPAL_PMIX_SESSION; // share only with procs in same session + rng->data.integer = OPAL_PMIX_RANGE_SESSION; // share only with procs in same session opal_list_append(&values, &rng->super); } else { /* unrecognized scope */ diff --git a/ompi/mpi/c/unpublish_name.c b/ompi/mpi/c/unpublish_name.c index e40f20e1d3..019d7106fe 100644 --- a/ompi/mpi/c/unpublish_name.c +++ b/ompi/mpi/c/unpublish_name.c @@ -12,10 +12,10 @@ * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -96,13 +96,13 @@ int MPI_Unpublish_name(const char *service_name, MPI_Info info, rng = OBJ_NEW(opal_value_t); rng->key = strdup(OPAL_PMIX_RANGE); rng->type = OPAL_INT; - rng->data.integer = OPAL_PMIX_NAMESPACE; // share only with procs in same nspace + rng->data.integer = OPAL_PMIX_RANGE_NAMESPACE; // share only with procs in same nspace opal_list_append(&pinfo, &rng->super); } else if (0 == strcmp(range, "session")) { rng = OBJ_NEW(opal_value_t); rng->key = strdup(OPAL_PMIX_RANGE); rng->type = OPAL_INT; - rng->data.integer = OPAL_PMIX_SESSION; // share only with procs in same session + rng->data.integer = OPAL_PMIX_RANGE_SESSION; // share only with procs in same session opal_list_append(&pinfo, &rng->super); } else { /* unrecognized scope */ diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 286ee2b33f..dc6df8671b 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -380,6 +380,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) char *cmd=NULL, *av=NULL; ompi_errhandler_errtrk_t errtrk; volatile bool active; + opal_list_t info; + opal_value_t *kv; OPAL_TIMING_DECLARE(tm); OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY); @@ -522,10 +524,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) /* Register the default errhandler callback */ errtrk.status = OPAL_ERROR; errtrk.active = true; - opal_pmix.register_errhandler(NULL, ompi_errhandler_callback, - ompi_errhandler_registration_callback, - (void*)&errtrk); + /* we want to go first */ + OBJ_CONSTRUCT(&info, opal_list_t); + kv = OBJ_NEW(opal_value_t); + kv->key = strdup(OPAL_PMIX_EVENT_ORDER_PREPEND); + opal_list_append(&info, &kv->super); + opal_pmix.register_evhandler(NULL, &info, ompi_errhandler_callback, + ompi_errhandler_registration_callback, + (void*)&errtrk); OMPI_WAIT_FOR_COMPLETION(errtrk.active); + OPAL_LIST_DESTRUCT(&info); if (OPAL_SUCCESS != errtrk.status) { error = "Error handler registration"; ret = errtrk.status; diff --git a/opal/Makefile.am b/opal/Makefile.am index e978e7dc5c..426091812e 100644 --- a/opal/Makefile.am +++ b/opal/Makefile.am @@ -10,7 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2015 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2016 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -76,7 +76,6 @@ nobase_opal_HEADERS = $(headers) endif include class/Makefile.am -include errhandler/Makefile.am include memoryhooks/Makefile.am include runtime/Makefile.am include threads/Makefile.am diff --git a/opal/errhandler/Makefile.am b/opal/errhandler/Makefile.am deleted file mode 100644 index b6e3eab4d5..0000000000 --- a/opal/errhandler/Makefile.am +++ /dev/null @@ -1,17 +0,0 @@ -# -*- makefile -*- -# -# Copyright (c) 2015 Intel, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# This makefile.am does not stand on its own - it is included from opal/Makefile.am - -headers += \ - errhandler/opal_errhandler.h - -lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \ - errhandler/opal_errhandler.c diff --git a/opal/errhandler/opal_errhandler.c b/opal/errhandler/opal_errhandler.c deleted file mode 100644 index fdd00d6767..0000000000 --- a/opal/errhandler/opal_errhandler.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2015 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/errhandler/opal_errhandler.h" - -opal_errhandler_fn_t errhandler = NULL; -void *cbdata = NULL; - -void opal_register_errhandler(opal_errhandler_fn_t newerr, void *cbd) -{ - errhandler = newerr; - cbdata = cbd; -} - -void opal_deregister_errhandler(void) -{ - errhandler = NULL; - cbdata = NULL; -} - -void opal_invoke_errhandler(int status, opal_proc_t *proc) -{ - if (NULL != errhandler) { - errhandler(status, proc, cbdata); - } -} diff --git a/opal/errhandler/opal_errhandler.h b/opal/errhandler/opal_errhandler.h deleted file mode 100644 index 4a1646f52b..0000000000 --- a/opal/errhandler/opal_errhandler.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2015 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OPAL_ERRHANDLER_H -#define OPAL_ERRHANDLER_H - -#include "opal_config.h" - -#include "opal/util/proc.h" - -typedef void (*opal_errhandler_fn_t)(int status, opal_proc_t *proc, void *cbdata); - -OPAL_DECLSPEC void opal_register_errhandler(opal_errhandler_fn_t errhandler, void *cbdata); - -OPAL_DECLSPEC void opal_deregister_errhandler(void); - -OPAL_DECLSPEC void opal_invoke_errhandler(int status, opal_proc_t *proc); - -#endif diff --git a/opal/include/opal/constants.h b/opal/include/opal/constants.h index bb0a575d50..89735ccb4d 100644 --- a/opal/include/opal/constants.h +++ b/opal/include/opal/constants.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -81,7 +81,8 @@ enum { OPAL_ERR_COMM_FAILURE = (OPAL_ERR_BASE - 51), OPAL_ERR_SERVER_NOT_AVAIL = (OPAL_ERR_BASE - 52), OPAL_ERR_IN_PROCESS = (OPAL_ERR_BASE - 53), - OPAL_ERR_DEBUGGER_RELEASE = (OPAL_ERR_BASE - 54) + OPAL_ERR_DEBUGGER_RELEASE = (OPAL_ERR_BASE - 54), + OPAL_ERR_HANDLERS_COMPLETE = (OPAL_ERR_BASE - 55) }; #define OPAL_ERR_MAX (OPAL_ERR_BASE - 100) diff --git a/opal/mca/pmix/base/base.h b/opal/mca/pmix/base/base.h index c1aec6e4e3..dd64912c4a 100644 --- a/opal/mca/pmix/base/base.h +++ b/opal/mca/pmix/base/base.h @@ -32,17 +32,23 @@ OPAL_DECLSPEC int opal_pmix_base_select(void); OPAL_DECLSPEC extern bool opal_pmix_base_allow_delayed_server; -OPAL_DECLSPEC void opal_pmix_base_register_handler(opal_list_t *info, - opal_pmix_notification_fn_t errhandler, - opal_pmix_errhandler_reg_cbfunc_t cbfunc, +OPAL_DECLSPEC void opal_pmix_base_register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, void *cbdata); -OPAL_DECLSPEC void opal_pmix_base_deregister_handler(int errhandler, +OPAL_DECLSPEC void opal_pmix_base_deregister_handler(size_t errhandler, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_DECLSPEC void opal_pmix_base_errhandler(int status, - opal_list_t *procs, - opal_list_t *info, - opal_pmix_release_cbfunc_t cbfunc, void *cbdata); +OPAL_DECLSPEC int opal_pmix_base_notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_DECLSPEC void opal_pmix_base_evhandler(int status, + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, void *cbdata); OPAL_DECLSPEC int opal_pmix_base_exchange(opal_value_t *info, opal_pmix_pdata_t *pdat, int timeout); diff --git a/opal/mca/pmix/base/pmix_base_fns.c b/opal/mca/pmix/base/pmix_base_fns.c index 57e873385a..d0df4043bd 100644 --- a/opal/mca/pmix/base/pmix_base_fns.c +++ b/opal/mca/pmix/base/pmix_base_fns.c @@ -46,39 +46,49 @@ void opal_pmix_base_set_evbase(opal_event_base_t *evbase) /******** ERRHANDLER SUPPORT FOR COMPONENTS THAT ******** DO NOT NATIVELY SUPPORT IT ********/ -static opal_pmix_notification_fn_t errhandler = NULL; +static opal_pmix_notification_fn_t evhandler = NULL; -void opal_pmix_base_register_handler(opal_list_t *info, +void opal_pmix_base_register_handler(opal_list_t *event_codes, + opal_list_t *info, opal_pmix_notification_fn_t err, - opal_pmix_errhandler_reg_cbfunc_t cbfunc, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, void *cbdata) { - errhandler = err; + evhandler = err; if (NULL != cbfunc) { cbfunc(OPAL_SUCCESS, 0, cbdata); } } -void opal_pmix_base_errhandler(int status, - opal_list_t *procs, - opal_list_t *info, - opal_pmix_release_cbfunc_t cbfunc, void *cbdata) +void opal_pmix_base_evhandler(int status, + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, void *cbdata) { - if (NULL != errhandler) { - errhandler(status, procs, info, cbfunc, cbdata); + if (NULL != evhandler) { + evhandler(status, source, info, results, cbfunc, cbdata); } } -void opal_pmix_base_deregister_handler(int errid, +void opal_pmix_base_deregister_handler(size_t errid, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { - errhandler = NULL; + evhandler = NULL; if (NULL != cbfunc) { cbfunc(OPAL_SUCCESS, cbdata); } } +int opal_pmix_base_notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return OPAL_SUCCESS; +} + struct lookup_caddy_t { volatile bool active; int status; diff --git a/opal/mca/pmix/cray/pmix_cray.c b/opal/mca/pmix/cray/pmix_cray.c index 97b5d0e8cc..8c86888ceb 100644 --- a/opal/mca/pmix/cray/pmix_cray.c +++ b/opal/mca/pmix/cray/pmix_cray.c @@ -110,8 +110,8 @@ const opal_pmix_base_module_t opal_pmix_cray_module = { .resolve_peers = cray_resolve_peers, .resolve_nodes = cray_resolve_nodes, .get_version = cray_get_version, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, + .register_evhandler = opal_pmix_base_register_handler, + .deregister_evhandler = opal_pmix_base_deregister_handler, .store_local = cray_store_local, .get_nspace = cray_get_nspace, .register_jobid = cray_register_jobid diff --git a/opal/mca/pmix/ext114/pmix_ext.c b/opal/mca/pmix/ext114/pmix_ext.c index 0d8d312153..860fa35bc9 100644 --- a/opal/mca/pmix/ext114/pmix_ext.c +++ b/opal/mca/pmix/ext114/pmix_ext.c @@ -85,11 +85,11 @@ const opal_pmix_base_module_t opal_pmix_ext114_module = { .server_deregister_client = pmix1_server_deregister_client, .server_setup_fork = pmix1_server_setup_fork, .server_dmodex_request = pmix1_server_dmodex, - .server_notify_error = pmix1_server_notify_error, + .server_notify_event = pmix1_server_notify_error, /* utility APIs */ .get_version = PMIx_Get_version, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, + .register_evhandler = opal_pmix_base_register_handler, + .deregister_evhandler = opal_pmix_base_deregister_handler, .store_local = pmix1_store_local, .get_nspace = pmix1_get_nspace, .register_jobid = pmix1_register_jobid diff --git a/opal/mca/pmix/ext114/pmix_ext.h b/opal/mca/pmix/ext114/pmix_ext.h index 44d87ecb38..a5ec4bad0b 100644 --- a/opal/mca/pmix/ext114/pmix_ext.h +++ b/opal/mca/pmix/ext114/pmix_ext.h @@ -145,19 +145,22 @@ OPAL_MODULE_DECLSPEC int pmix1_server_register_nspace(opal_jobid_t jobid, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC void pmix1_server_deregister_nspace(opal_jobid_t jobid); +OPAL_MODULE_DECLSPEC void pmix1_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); OPAL_MODULE_DECLSPEC int pmix1_server_register_client(const opal_process_name_t *proc, uid_t uid, gid_t gid, void *server_object, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC void pmix1_server_deregister_client(const opal_process_name_t *proc); +OPAL_MODULE_DECLSPEC void pmix1_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); OPAL_MODULE_DECLSPEC int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env); OPAL_MODULE_DECLSPEC int pmix1_server_dmodex(const opal_process_name_t *proc, opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); OPAL_MODULE_DECLSPEC int pmix1_server_notify_error(int status, - opal_list_t *procs, - opal_list_t *error_procs, + const opal_process_name_t *source, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); diff --git a/opal/mca/pmix/ext114/pmix_ext_client.c b/opal/mca/pmix/ext114/pmix_ext_client.c index 81b193a7e7..388347c3b0 100644 --- a/opal/mca/pmix/ext114/pmix_ext_client.c +++ b/opal/mca/pmix/ext114/pmix_ext_client.c @@ -43,8 +43,10 @@ static int errhdler_ref = 0; } while (0) -static void completion_handler (void * cbdata) { - int * cond = (int *)cbdata; +static void completion_handler(int status, opal_list_t *results, + opal_pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata) { + int * cond = (int *)notification_cbdata; *cond = 0; } @@ -81,7 +83,7 @@ static void myerr(pmix_status_t status, } /* call the base errhandler */ - opal_pmix_base_errhandler(rc, &plist, &ilist, completion_handler, (void *)&cond); + opal_pmix_base_evhandler(rc, &OPAL_PROC_MY_NAME, &plist, &ilist, completion_handler, (void *)&cond); PMIX_WAIT_FOR_COMPLETION(cond); OPAL_LIST_DESTRUCT(&plist); diff --git a/opal/mca/pmix/ext114/pmix_ext_server_north.c b/opal/mca/pmix/ext114/pmix_ext_server_north.c index 8b75dcf701..d272fb014e 100644 --- a/opal/mca/pmix/ext114/pmix_ext_server_north.c +++ b/opal/mca/pmix/ext114/pmix_ext_server_north.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Mellanox Technologies, Inc. @@ -44,7 +44,7 @@ /* These are the interfaces used by the embedded PMIx server * to call up into ORTE for service requests */ -static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object); + static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object); static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object, @@ -85,20 +85,20 @@ static pmix_status_t server_listener_fn(int listening_sd, pmix_connection_cbfunc_t cbfunc); pmix_server_module_t mymodule = { - server_client_connected_fn, - server_client_finalized_fn, - server_abort_fn, - server_fencenb_fn, - server_dmodex_req_fn, - server_publish_fn, - server_lookup_fn, - server_unpublish_fn, - server_spawn_fn, - server_connect_fn, - server_disconnect_fn, - server_register_events, - server_deregister_events, - server_listener_fn + .client_connected = server_client_connected_fn, + .client_finalized = server_client_finalized_fn, + .abort = server_abort_fn, + .fence_nb = server_fencenb_fn, + .direct_modex = server_dmodex_req_fn, + .publish = server_publish_fn, + .lookup = server_lookup_fn, + .unpublish = server_unpublish_fn, + .spawn = server_spawn_fn, + .connect = server_connect_fn, + .disconnect = server_disconnect_fn, + .register_events = server_register_events, + .deregister_events = server_deregister_events, + .listener = server_listener_fn }; opal_pmix_server_module_t *host_module = NULL; @@ -130,7 +130,8 @@ static pmix_status_t server_client_connected_fn(const pmix_proc_t *p, void *serv proc.vpid = p->rank; /* pass it up */ - rc = host_module->client_connected(&proc, server_object); + rc = host_module->client_connected(&proc, server_object, + NULL, NULL); return pmix1_convert_opalrc(rc); } diff --git a/opal/mca/pmix/ext114/pmix_ext_server_south.c b/opal/mca/pmix/ext114/pmix_ext_server_south.c index 31e1f18299..0cfb84bd88 100644 --- a/opal/mca/pmix/ext114/pmix_ext_server_south.c +++ b/opal/mca/pmix/ext114/pmix_ext_server_south.c @@ -50,8 +50,10 @@ extern opal_pmix_server_module_t *host_module; static char *dbgvalue=NULL; static int errhdler_ref = 0; -static void completion_handler (void * cbdata) { - int * cond = (int *)cbdata; +static void completion_handler(int status, opal_list_t *results, + opal_pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata) { + int * cond = (int *)notification_cbdata; *cond = 0; } @@ -95,7 +97,7 @@ static void myerr(pmix_status_t status, } /* call the base errhandler */ - opal_pmix_base_errhandler(rc, &plist, &ilist, completion_handler, (void *)&cond); + opal_pmix_base_evhandler(rc, &OPAL_PROC_MY_NAME, &plist, &ilist, completion_handler, (void *)&cond); PMIX_WAIT_FOR_COMPLETION(cond); OPAL_LIST_DESTRUCT(&plist); @@ -263,7 +265,9 @@ int pmix1_server_register_nspace(opal_jobid_t jobid, return pmix1_convert_rc(rc); } -void pmix1_server_deregister_nspace(opal_jobid_t jobid) +void pmix1_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) { opal_pmix1_jobid_trkr_t *jptr; @@ -306,7 +310,9 @@ int pmix1_server_register_client(const opal_process_name_t *proc, return pmix1_convert_rc(rc); } -void pmix1_server_deregister_client(const opal_process_name_t *proc) +void pmix1_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) { opal_pmix1_jobid_trkr_t *jptr; pmix_proc_t p; @@ -376,46 +382,20 @@ int pmix1_server_dmodex(const opal_process_name_t *proc, } int pmix1_server_notify_error(int status, - opal_list_t *procs, - opal_list_t *error_procs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata) + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { opal_value_t *kv; pmix_info_t *pinfo; - size_t sz, psz, esz, n; - pmix_proc_t *ps, *eps; + size_t sz, n; pmix_status_t rc; pmix1_opcaddy_t *op; - opal_namelist_t *nm; - /* convert the list of procs */ - if (NULL != procs) { - psz = opal_list_get_size(procs); - PMIX_PROC_CREATE(ps, psz); - n = 0; - OPAL_LIST_FOREACH(nm, procs, opal_namelist_t) { - (void)opal_snprintf_jobid(ps[n].nspace, PMIX_MAX_NSLEN, nm->name.jobid); - ps[n].rank = (int)nm->name.vpid; - ++n; - } - } else { - psz = 0; - ps = NULL; - } - if (NULL != error_procs) { - esz = opal_list_get_size(error_procs); - PMIX_PROC_CREATE(eps, esz); - n = 0; - OPAL_LIST_FOREACH(nm, error_procs, opal_namelist_t) { - (void)opal_snprintf_jobid(eps[n].nspace, PMIX_MAX_NSLEN, nm->name.jobid); - eps[n].rank = (int)nm->name.vpid; - ++n; - } - } else { - esz = 0; - eps = NULL; - } + /* setup the caddy */ + op = OBJ_NEW(pmix1_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; /* convert the list to an array of pmix_info_t */ if (NULL != info) { @@ -430,21 +410,12 @@ int pmix1_server_notify_error(int status, sz = 0; pinfo = NULL; } - - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->procs = ps; - op->nprocs = psz; - op->error_procs = eps; - op->nerror_procs = esz; op->info = pinfo; op->sz = sz; - op->opcbfunc = cbfunc; - op->cbdata = cbdata; rc = pmix1_convert_opalrc(status); - rc = PMIx_Notify_error(rc, ps, psz, eps, esz, - pinfo, sz, opcbfunc, op); + rc = PMIx_Notify_error(rc, NULL, 0, NULL, 0, + pinfo, sz, opcbfunc, op); if (PMIX_SUCCESS != rc) { OBJ_RELEASE(op); } diff --git a/opal/mca/pmix/ext20/pmix_ext20.c b/opal/mca/pmix/ext20/pmix_ext20.c index 11c939d2a6..b14231c9d7 100644 --- a/opal/mca/pmix/ext20/pmix_ext20.c +++ b/opal/mca/pmix/ext20/pmix_ext20.c @@ -45,61 +45,75 @@ /* These are functions used by both client and server to * access common functions in the embedded PMIx library */ -static const char *pmix1_get_nspace(opal_jobid_t jobid); -static void pmix1_register_jobid(opal_jobid_t jobid, const char *nspace); +static const char *pmix20_get_nspace(opal_jobid_t jobid); +static void pmix20_register_jobid(opal_jobid_t jobid, const char *nspace); +static void register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, + void *cbdata); +static void deregister_handler(size_t evhandler, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +static int notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); const opal_pmix_base_module_t opal_pmix_ext20_module = { /* client APIs */ - .init = pmix1_client_init, - .finalize = pmix1_client_finalize, - .initialized = pmix1_initialized, - .abort = pmix1_abort, - .commit = pmix1_commit, - .fence = pmix1_fence, - .fence_nb = pmix1_fencenb, - .put = pmix1_put, - .get = pmix1_get, - .get_nb = pmix1_getnb, - .publish = pmix1_publish, - .publish_nb = pmix1_publishnb, - .lookup = pmix1_lookup, - .lookup_nb = pmix1_lookupnb, - .unpublish = pmix1_unpublish, - .unpublish_nb = pmix1_unpublishnb, - .spawn = pmix1_spawn, - .spawn_nb = pmix1_spawnnb, - .connect = pmix1_connect, - .connect_nb = pmix1_connectnb, - .disconnect = pmix1_disconnect, - .disconnect_nb = pmix1_disconnectnb, - .resolve_peers = pmix1_resolve_peers, - .resolve_nodes = pmix1_resolve_nodes, + .init = pmix20_client_init, + .finalize = pmix20_client_finalize, + .initialized = pmix20_initialized, + .abort = pmix20_abort, + .commit = pmix20_commit, + .fence = pmix20_fence, + .fence_nb = pmix20_fencenb, + .put = pmix20_put, + .get = pmix20_get, + .get_nb = pmix20_getnb, + .publish = pmix20_publish, + .publish_nb = pmix20_publishnb, + .lookup = pmix20_lookup, + .lookup_nb = pmix20_lookupnb, + .unpublish = pmix20_unpublish, + .unpublish_nb = pmix20_unpublishnb, + .spawn = pmix20_spawn, + .spawn_nb = pmix20_spawnnb, + .connect = pmix20_connect, + .connect_nb = pmix20_connectnb, + .disconnect = pmix20_disconnect, + .disconnect_nb = pmix20_disconnectnb, + .resolve_peers = pmix20_resolve_peers, + .resolve_nodes = pmix20_resolve_nodes, /* server APIs */ - .server_init = pmix1_server_init, - .server_finalize = pmix1_server_finalize, - .generate_regex = pmix1_server_gen_regex, - .generate_ppn = pmix1_server_gen_ppn, - .server_register_nspace = pmix1_server_register_nspace, - .server_deregister_nspace = pmix1_server_deregister_nspace, - .server_register_client = pmix1_server_register_client, - .server_deregister_client = pmix1_server_deregister_client, - .server_setup_fork = pmix1_server_setup_fork, - .server_dmodex_request = pmix1_server_dmodex, - .server_notify_error = pmix1_server_notify_error, + .server_init = pmix20_server_init, + .server_finalize = pmix20_server_finalize, + .generate_regex = pmix20_server_gen_regex, + .generate_ppn = pmix20_server_gen_ppn, + .server_register_nspace = pmix20_server_register_nspace, + .server_deregister_nspace = pmix20_server_deregister_nspace, + .server_register_client = pmix20_server_register_client, + .server_deregister_client = pmix20_server_deregister_client, + .server_setup_fork = pmix20_server_setup_fork, + .server_dmodex_request = pmix20_server_dmodex, + .server_notify_event = pmix20_server_notify_event, /* utility APIs */ .get_version = PMIx_Get_version, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, - .store_local = pmix1_store_local, - .get_nspace = pmix1_get_nspace, - .register_jobid = pmix1_register_jobid + .register_evhandler = register_handler, + .deregister_evhandler = deregister_handler, + .notify_event = notify_event, + .store_local = pmix20_store_local, + .get_nspace = pmix20_get_nspace, + .register_jobid = pmix20_register_jobid }; -static const char *pmix1_get_nspace(opal_jobid_t jobid) +static const char *pmix20_get_nspace(opal_jobid_t jobid) { - opal_pmix1_jobid_trkr_t *jptr; + opal_pmix20_jobid_trkr_t *jptr; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == jobid) { return jptr->nspace; } @@ -107,25 +121,302 @@ static const char *pmix1_get_nspace(opal_jobid_t jobid) return NULL; } -static void pmix1_register_jobid(opal_jobid_t jobid, const char *nspace) +static void pmix20_register_jobid(opal_jobid_t jobid, const char *nspace) { - opal_pmix1_jobid_trkr_t *jptr; + opal_pmix20_jobid_trkr_t *jptr; /* if we don't already have it, add this to our jobid tracker */ - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == jobid) { return; } } - jptr = OBJ_NEW(opal_pmix1_jobid_trkr_t); + jptr = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(jptr->nspace, nspace, PMIX_MAX_NSLEN); jptr->jobid = jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &jptr->super); } -pmix_status_t pmix1_convert_opalrc(int rc) +static void completion_handler(int status, void *cbdata) +{ + opal_pmix20_event_chain_t *chain = (opal_pmix20_event_chain_t*)cbdata; + if (NULL != chain->info) { + OPAL_LIST_RELEASE(chain->info); + } +} + +static void progress_local_event_hdlr(int status, + opal_list_t *results, + opal_pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata) +{ + opal_pmix20_event_chain_t *chain = (opal_pmix20_event_chain_t*)notification_cbdata; + size_t n; + opal_list_item_t *nxt; + opal_pmix20_single_event_t *sing; + opal_pmix20_multi_event_t *multi; + opal_pmix20_default_event_t *def; + + /* if any results were provided, then add them here */ + if (NULL != results) { + while (NULL != (nxt = opal_list_remove_first(results))) { + opal_list_append(results, nxt); + } + } + + /* if the caller indicates that the chain is completed, then stop here */ + if (OPAL_ERR_HANDLERS_COMPLETE == status) { + goto complete; + } + + /* see if we need to continue, starting with the single code events */ + if (NULL != chain->sing) { + /* the last handler was for a single code - see if there are + * any others that match this event */ + while (opal_list_get_end(&mca_pmix_ext20_component.single_events) != (nxt = opal_list_get_next(&chain->sing->super))) { + sing = (opal_pmix20_single_event_t*)nxt; + if (sing->code == chain->status) { + OBJ_RETAIN(chain); + chain->sing = sing; + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + goto complete; + } + } + /* if we get here, then there are no more single code + * events that match */ + chain->sing = NULL; + /* pickup the beginning of the multi-code event list */ + if (0 < opal_list_get_size(&mca_pmix_ext20_component.multi_events)) { + chain->multi = (opal_pmix20_multi_event_t*)opal_list_get_begin(&mca_pmix_ext20_component.multi_events); + } + } + + /* see if we need to continue with the multi code events */ + if (NULL != chain->multi) { + while (opal_list_get_end(&mca_pmix_ext20_component.multi_events) != (nxt = opal_list_get_next(&chain->multi->super))) { + multi = (opal_pmix20_multi_event_t*)nxt; + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + OBJ_RETAIN(chain); + chain->multi = multi; + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + goto complete; + } + } + } + /* if we get here, then there are no more multi-mode + * events that match */ + chain->multi = NULL; + /* pickup the beginning of the default event list */ + if (0 < opal_list_get_size(&mca_pmix_ext20_component.default_events)) { + chain->def = (opal_pmix20_default_event_t*)opal_list_get_begin(&mca_pmix_ext20_component.default_events); + } + } + + /* if they didn't want it to go to a default handler, then we are done */ + if (chain->nondefault) { + goto complete; + } + + if (NULL != chain->def) { + if (opal_list_get_end(&mca_pmix_ext20_component.default_events) != (nxt = opal_list_get_next(&chain->def->super))) { + def = (opal_pmix20_default_event_t*)nxt; + OBJ_RETAIN(chain); + chain->def = def; + def->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + } + } + + complete: + /* we still have to call their final callback */ + if (NULL != chain->final_cbfunc) { + chain->final_cbfunc(OPAL_SUCCESS, chain->final_cbdata); + } + /* maintain acctng */ + OBJ_RELEASE(chain); + /* let the caller know that we are done with their callback */ + if (NULL != cbfunc) { + cbfunc(OPAL_SUCCESS, thiscbdata); + } +} + +static void _event_hdlr(int sd, short args, void *cbdata) +{ + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + size_t n; + opal_pmix20_event_chain_t *chain; + opal_pmix20_single_event_t *sing; + opal_pmix20_multi_event_t *multi; + opal_pmix20_default_event_t *def; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s RECEIVED NOTIFICATION OF STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), cd->status); + + chain = OBJ_NEW(opal_pmix20_event_chain_t); + /* point it at our final callback */ + chain->final_cbfunc = completion_handler; + chain->final_cbdata = chain; + + /* carry across provided info */ + chain->status = cd->status; + chain->source = cd->pname; + chain->info = cd->info; + chain->nondefault = cd->nondefault; + + /* cycle thru the single-event registrations first */ + OPAL_LIST_FOREACH(sing, &mca_pmix_ext20_component.single_events, opal_pmix20_single_event_t) { + if (sing->code == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + OBJ_RETAIN(chain); + chain->sing = sing; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CALLING SINGLE EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + return; + } + } + + /* if we didn't find any match in the single-event registrations, + * then cycle thru the multi-event registrations next */ + OPAL_LIST_FOREACH(multi, &mca_pmix_ext20_component.multi_events, opal_pmix20_multi_event_t) { + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + OBJ_RETAIN(chain); + chain->multi = multi; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CALLING MULTI EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + return; + } + } + } + + /* if they didn't want it to go to a default handler, then we are done */ + if (chain->nondefault) { + /* if we get here, then we need to cache this event in case they + * register for it later - we cannot lose individual events */ + opal_list_append(&mca_pmix_ext20_component.cache, &chain->super); + return; + } + + /* we are done with the threadshift caddy */ + OBJ_RELEASE(cd); + + /* finally, pass it to any default handlers */ + if (0 < opal_list_get_size(&mca_pmix_ext20_component.default_events)) { + def = (opal_pmix20_default_event_t*)opal_list_get_first(&mca_pmix_ext20_component.default_events); + OBJ_RETAIN(chain); + chain->def = def; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CALLING DEFAULT EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + def->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + return; + } + + /* we still have to call their final callback */ + if (NULL != chain->final_cbfunc) { + chain->final_cbfunc(PMIX_SUCCESS, chain->final_cbdata); + } + return; +} + +/* this function will be called by the PMIx client library + * whenever it receives notification of an event. The + * notification can come from an ORTE daemon (when launched + * by mpirun), directly from a RM (when direct launched), or + * from another process (via the local daemon). + * The call will occur in the PMIx event base */ +void pmix20_event_hdlr(size_t evhdlr_registration_id, + pmix_status_t status, const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + pmix20_threadshift_t *cd; + int rc; + opal_value_t *iptr; + size_t n; + + /* this is in the PMIx local thread - need to threadshift to + * our own thread as we will be accessing framework-global + * lists and objects */ + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s RECEIVED NOTIFICATION OF STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), status); + + cd = OBJ_NEW(pmix20_threadshift_t); + + /* convert the incoming status */ + cd->status = pmix20_convert_rc(status); + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CONVERTED STATUS %d TO STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), status, cd->status); + /* convert the nspace/rank to an opal_process_name_t */ + if (NULL == source) { + cd->pname.jobid = OPAL_NAME_INVALID->jobid; + cd->pname.vpid = OPAL_NAME_INVALID->vpid; + } else { + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&cd->pname.jobid, source->nspace))) { + OPAL_ERROR_LOG(rc); + OBJ_RELEASE(cd); + return; + } + cd->pname.vpid = source->rank; + } + + /* convert the array of info */ + if (NULL != info) { + cd->info = OBJ_NEW(opal_list_t); + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + cd->nondefault = true; + } + iptr = OBJ_NEW(opal_value_t); + iptr->key = strdup(info[n].key); + pmix20_value_unload(iptr, &info[n].value); + opal_list_append(cd->info, &iptr->super); + } + } + /* now push it into the local thread */ + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _event_hdlr, cd); + event_active(&cd->ev, EV_WRITE, 1); + + /* we don't need any of the data they provided, + * so let them go */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } +} + +pmix_status_t pmix20_convert_opalrc(int rc) { switch (rc) { + case OPAL_ERR_DEBUGGER_RELEASE: + return PMIX_ERR_DEBUGGER_RELEASE; + case OPAL_ERR_NOT_IMPLEMENTED: case OPAL_ERR_NOT_SUPPORTED: return PMIX_ERR_NOT_SUPPORTED; @@ -165,9 +456,12 @@ pmix_status_t pmix1_convert_opalrc(int rc) } } -int pmix1_convert_rc(pmix_status_t rc) +int pmix20_convert_rc(pmix_status_t rc) { switch (rc) { + case PMIX_ERR_DEBUGGER_RELEASE: + return OPAL_ERR_DEBUGGER_RELEASE; + case PMIX_ERR_NOT_SUPPORTED: return OPAL_ERR_NOT_SUPPORTED; @@ -210,7 +504,7 @@ int pmix1_convert_rc(pmix_status_t rc) } } -pmix_scope_t pmix1_convert_opalscope(opal_pmix_scope_t scope) { +pmix_scope_t pmix20_convert_opalscope(opal_pmix_scope_t scope) { switch(scope) { case OPAL_PMIX_LOCAL: return PMIX_LOCAL; @@ -223,13 +517,51 @@ pmix_scope_t pmix1_convert_opalscope(opal_pmix_scope_t scope) { } } -void pmix1_value_load(pmix_value_t *v, - opal_value_t *kv) +pmix_data_range_t pmix20_convert_opalrange(opal_pmix_data_range_t range) { + switch(range) { + case OPAL_PMIX_RANGE_UNDEF: + return PMIX_RANGE_UNDEF; + case OPAL_PMIX_RANGE_LOCAL: + return PMIX_RANGE_LOCAL; + case OPAL_PMIX_RANGE_NAMESPACE: + return PMIX_RANGE_NAMESPACE; + case OPAL_PMIX_RANGE_SESSION: + return PMIX_RANGE_SESSION; + case OPAL_PMIX_RANGE_GLOBAL: + return PMIX_RANGE_GLOBAL; + case OPAL_PMIX_RANGE_CUSTOM: + return PMIX_RANGE_CUSTOM; + default: + return PMIX_SCOPE_UNDEF; + } +} + +opal_pmix_data_range_t pmix20_convert_range(pmix_data_range_t range) { + switch(range) { + case PMIX_RANGE_UNDEF: + return OPAL_PMIX_RANGE_UNDEF; + case PMIX_RANGE_LOCAL: + return OPAL_PMIX_RANGE_LOCAL; + case PMIX_RANGE_NAMESPACE: + return OPAL_PMIX_RANGE_NAMESPACE; + case PMIX_RANGE_SESSION: + return OPAL_PMIX_RANGE_SESSION; + case PMIX_RANGE_GLOBAL: + return OPAL_PMIX_RANGE_GLOBAL; + case PMIX_RANGE_CUSTOM: + return OPAL_PMIX_RANGE_CUSTOM; + default: + return OPAL_PMIX_SCOPE_UNDEF; + } +} + + +void pmix20_value_load(pmix_value_t *v, + opal_value_t *kv) { switch(kv->type) { case OPAL_UNDEF: v->type = PMIX_UNDEF; - opal_output(0, "TYPE WAS UNDEF"); break; case OPAL_BOOL: v->type = PMIX_BOOL; @@ -324,7 +656,7 @@ void pmix1_value_load(pmix_value_t *v, } } -int pmix1_value_unload(opal_value_t *kv, +int pmix20_value_unload(opal_value_t *kv, const pmix_value_t *v) { int rc=OPAL_SUCCESS; @@ -427,13 +759,341 @@ int pmix1_value_unload(opal_value_t *kv, return rc; } +static void _reg_hdlr(int sd, short args, void *cbdata) +{ + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + opal_pmix20_event_chain_t *chain; + opal_pmix20_single_event_t *sing = NULL; + opal_pmix20_multi_event_t *multi = NULL; + opal_pmix20_default_event_t *def = NULL; + opal_value_t *kv; + int i; + bool prepend = false; + size_t n; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s REGISTER HANDLER CODES %s", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + (NULL == cd->event_codes) ? "NULL" : "NON-NULL"); + + if (NULL != cd->info) { + OPAL_LIST_FOREACH(kv, cd->info, opal_value_t) { + if (0 == strcmp(kv->key, OPAL_PMIX_EVENT_ORDER_PREPEND)) { + prepend = true; + break; + } + } + } + + if (NULL == cd->event_codes) { + /* this is a default handler */ + def = OBJ_NEW(opal_pmix20_default_event_t); + def->handler = cd->evhandler; + def->index = mca_pmix_ext20_component.evindex; + if (prepend) { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s PREPENDING TO DEFAULT EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_prepend(&mca_pmix_ext20_component.default_events, &def->super); + } else { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s APPENDING TO DEFAULT EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_append(&mca_pmix_ext20_component.default_events, &def->super); + } + } else if (1 == opal_list_get_size(cd->event_codes)) { + /* single handler */ + sing = OBJ_NEW(opal_pmix20_single_event_t); + kv = (opal_value_t*)opal_list_get_first(cd->event_codes); + sing->code = kv->data.integer; + sing->index = mca_pmix_ext20_component.evindex; + sing->handler = cd->evhandler; + if (prepend) { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s PREPENDING TO SINGLE EVENTS WITH CODE %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), sing->code); + opal_list_prepend(&mca_pmix_ext20_component.single_events, &sing->super); + } else { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s APPENDING TO SINGLE EVENTS WITH CODE %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), sing->code); + opal_list_append(&mca_pmix_ext20_component.single_events, &sing->super); + } + } else { + multi = OBJ_NEW(opal_pmix20_multi_event_t); + multi->ncodes = opal_list_get_size(cd->event_codes); + multi->codes = (int*)malloc(multi->ncodes * sizeof(int)); + i=0; + OPAL_LIST_FOREACH(kv, cd->event_codes, opal_value_t) { + multi->codes[i] = kv->data.integer; + ++i; + } + multi->index = mca_pmix_ext20_component.evindex; + multi->handler = cd->evhandler; + if (prepend) { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s PREPENDING TO MULTI EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_prepend(&mca_pmix_ext20_component.multi_events, &multi->super); + } else { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s APPENDING TO MULTI EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_append(&mca_pmix_ext20_component.multi_events, &multi->super); + } + } + + /* release the caller */ + if (NULL != cd->cbfunc) { + cd->cbfunc(OPAL_SUCCESS, mca_pmix_ext20_component.evindex, cd->cbdata); + } + mca_pmix_ext20_component.evindex++; + + /* check if any matching notifications have been cached - only nondefault + * events will have been cached*/ + if (NULL == def) { + /* check single code registrations */ + if (NULL != sing) { + OPAL_LIST_FOREACH(chain, &mca_pmix_ext20_component.cache, opal_pmix20_event_chain_t) { + if (sing->code == chain->status) { + opal_list_remove_item(&mca_pmix_ext20_component.cache, &chain->super); + chain->sing = sing; + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + } else if (NULL != multi) { + /* check for multi code registrations */ + OPAL_LIST_FOREACH(chain, &mca_pmix_ext20_component.cache, opal_pmix20_event_chain_t) { + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + opal_list_remove_item(&mca_pmix_ext20_component.cache, &chain->super); + chain->multi = multi; + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + } + } + } + + OBJ_RELEASE(cd); + return; +} +static void register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, + void *cbdata) +{ + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + + OPAL_PMIX_THREADSHIFT(event_codes, info, evhandler, _reg_hdlr, cbfunc, cbdata); + return; +} + +static void _dereg_hdlr(int sd, short args, void *cbdata) +{ + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + opal_pmix20_single_event_t *sing; + opal_pmix20_multi_event_t *multi; + opal_pmix20_default_event_t *def; + + /* check the single events first */ + OPAL_LIST_FOREACH(sing, &mca_pmix_ext20_component.single_events, opal_pmix20_single_event_t) { + if (cd->handler == sing->index) { + opal_list_remove_item(&mca_pmix_ext20_component.single_events, &sing->super); + OBJ_RELEASE(sing); + goto release; + } + } + /* check multi events */ + OPAL_LIST_FOREACH(multi, &mca_pmix_ext20_component.multi_events, opal_pmix20_multi_event_t) { + if (cd->handler == multi->index) { + opal_list_remove_item(&mca_pmix_ext20_component.multi_events, &multi->super); + OBJ_RELEASE(multi); + goto release; + } + } + /* check default events */ + OPAL_LIST_FOREACH(def, &mca_pmix_ext20_component.default_events, opal_pmix20_default_event_t) { + if (cd->handler == def->index) { + opal_list_remove_item(&mca_pmix_ext20_component.default_events, &def->super); + OBJ_RELEASE(def); + break; + } + } + + release: + if (NULL != cd->opcbfunc) { + cd->opcbfunc(OPAL_SUCCESS, cd->cbdata); + } + OBJ_RELEASE(cd); +} + +static void deregister_handler(size_t evhandler, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + OPAL_PMIX_OP_THREADSHIFT(evhandler, _dereg_hdlr, cbfunc, cbdata); + return; +} + +static void _notify_event(int sd, short args, void *cbdata) +{ + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + size_t i; + opal_pmix20_single_event_t *sing; + opal_pmix20_multi_event_t *multi; + opal_pmix20_default_event_t *def; + opal_pmix20_event_chain_t *chain; + + /* check the single events first */ + OPAL_LIST_FOREACH(sing, &mca_pmix_ext20_component.single_events, opal_pmix20_single_event_t) { + if (cd->status == sing->code) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + chain = OBJ_NEW(opal_pmix20_event_chain_t); + chain->status = cd->status; + chain->range = pmix20_convert_opalrange(cd->range); + chain->source = *(cd->source); + chain->info = cd->info; + chain->final_cbfunc = cd->opcbfunc; + chain->final_cbdata = cd->cbdata; + chain->sing = sing; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "[%s] CALLING SINGLE EVHDLR FOR STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), chain->status); + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + /* check multi events */ + OPAL_LIST_FOREACH(multi, &mca_pmix_ext20_component.multi_events, opal_pmix20_multi_event_t) { + for (i=0; i < multi->ncodes; i++) { + if (cd->status == multi->codes[i]) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + chain = OBJ_NEW(opal_pmix20_event_chain_t); + chain->status = cd->status; + chain->range = pmix20_convert_opalrange(cd->range); + chain->source = *(cd->source); + chain->info = cd->info; + chain->final_cbfunc = cd->opcbfunc; + chain->final_cbdata = cd->cbdata; + chain->multi = multi; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "[%s] CALLING MULTI EVHDLR FOR STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), chain->status); + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + } + /* check default events */ + if (0 < opal_list_get_size(&mca_pmix_ext20_component.default_events)) { + def = (opal_pmix20_default_event_t*)opal_list_get_first(&mca_pmix_ext20_component.default_events); + chain = OBJ_NEW(opal_pmix20_event_chain_t); + chain->status = cd->status; + chain->range = pmix20_convert_opalrange(cd->range); + chain->source = *(cd->source); + chain->info = cd->info; + chain->final_cbfunc = cd->opcbfunc; + chain->final_cbdata = cd->cbdata; + chain->def = def; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "[%s] CALLING DEFAULT EVHDLR FOR STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), chain->status); + def->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + + /* if we get here, then there are no registered event handlers */ + if (NULL != cd->opcbfunc) { + cd->opcbfunc(OPAL_ERR_NOT_FOUND, cd->cbdata); + } + OBJ_RELEASE(cd); + return; +} + +static int notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + OPAL_PMIX_NOTIFY_THREADSHIFT(status, source, range, info, _notify_event, cbfunc, cbdata); + return OPAL_SUCCESS; +} /**** INSTANTIATE INTERNAL CLASSES ****/ -OBJ_CLASS_INSTANCE(opal_pmix1_jobid_trkr_t, +OBJ_CLASS_INSTANCE(opal_pmix20_jobid_trkr_t, opal_list_item_t, NULL, NULL); -static void opcon(pmix1_opcaddy_t *p) +OBJ_CLASS_INSTANCE(opal_pmix20_single_event_t, + opal_list_item_t, + NULL, NULL); + +static void mtevcon(opal_pmix20_multi_event_t *p) +{ + p->codes = NULL; + p->ncodes = 0; +} +static void mtevdes(opal_pmix20_multi_event_t *p) +{ + if (NULL != p->codes) { + free(p->codes); + } +} +OBJ_CLASS_INSTANCE(opal_pmix20_multi_event_t, + opal_list_item_t, + mtevcon, mtevdes); + +OBJ_CLASS_INSTANCE(opal_pmix20_default_event_t, + opal_list_item_t, + NULL, NULL); + +static void chcon(opal_pmix20_event_chain_t *p) +{ + p->nondefault = false; + p->info = NULL; + OBJ_CONSTRUCT(&p->results, opal_list_t); + p->sing = NULL; + p->multi = NULL; + p->def = NULL; + p->final_cbfunc = NULL; + p->final_cbdata = NULL; +} +static void chdes(opal_pmix20_event_chain_t *p) +{ + OPAL_LIST_DESTRUCT(&p->results); +} +OBJ_CLASS_INSTANCE(opal_pmix20_event_chain_t, + opal_list_item_t, + chcon, chdes); + +static void opcon(pmix20_opcaddy_t *p) { memset(&p->p, 0, sizeof(pmix_proc_t)); p->procs = NULL; @@ -444,6 +1104,7 @@ static void opcon(pmix1_opcaddy_t *p) p->ninfo = 0; p->apps = NULL; p->sz = 0; + p->active = false; p->opcbfunc = NULL; p->mdxcbfunc = NULL; p->valcbfunc = NULL; @@ -451,7 +1112,7 @@ static void opcon(pmix1_opcaddy_t *p) p->spcbfunc = NULL; p->cbdata = NULL; } -static void opdes(pmix1_opcaddy_t *p) +static void opdes(pmix20_opcaddy_t *p) { if (NULL != p->procs) { PMIX_PROC_FREE(p->procs, p->nprocs); @@ -466,11 +1127,11 @@ static void opdes(pmix1_opcaddy_t *p) PMIX_APP_FREE(p->apps, p->sz); } } -OBJ_CLASS_INSTANCE(pmix1_opcaddy_t, +OBJ_CLASS_INSTANCE(pmix20_opcaddy_t, opal_object_t, opcon, opdes); -static void ocadcon(pmix1_opalcaddy_t *p) +static void ocadcon(pmix20_opalcaddy_t *p) { OBJ_CONSTRUCT(&p->procs, opal_list_t); OBJ_CONSTRUCT(&p->info, opal_list_t); @@ -484,12 +1145,27 @@ static void ocadcon(pmix1_opalcaddy_t *p) p->odmdxfunc = NULL; p->ocbdata = NULL; } -static void ocaddes(pmix1_opalcaddy_t *p) +static void ocaddes(pmix20_opalcaddy_t *p) { OPAL_LIST_DESTRUCT(&p->procs); OPAL_LIST_DESTRUCT(&p->info); OPAL_LIST_DESTRUCT(&p->apps); } -OBJ_CLASS_INSTANCE(pmix1_opalcaddy_t, +OBJ_CLASS_INSTANCE(pmix20_opalcaddy_t, opal_object_t, ocadcon, ocaddes); + +static void tscon(pmix20_threadshift_t *p) +{ + p->active = false; + p->source = NULL; + p->event_codes = NULL; + p->info = NULL; + p->evhandler = NULL; + p->cbfunc = NULL; + p->opcbfunc = NULL; + p->cbdata = NULL; +} +OBJ_CLASS_INSTANCE(pmix20_threadshift_t, + opal_object_t, + tscon, NULL); diff --git a/opal/mca/pmix/ext20/pmix_ext20.h b/opal/mca/pmix/ext20/pmix_ext20.h index 4d2a82cd9c..2b98c04967 100644 --- a/opal/mca/pmix/ext20/pmix_ext20.h +++ b/opal/mca/pmix/ext20/pmix_ext20.h @@ -23,13 +23,13 @@ #include #endif +#include "opal/class/opal_list.h" #include "opal/mca/mca.h" #include "opal/mca/event/event.h" #include "opal/util/proc.h" #include "opal/mca/pmix/pmix.h" #include "pmix_server.h" -#include "pmix_server.h" #include "pmix/pmix_common.h" BEGIN_C_DECLS @@ -38,6 +38,12 @@ typedef struct { opal_pmix_base_component_t super; opal_list_t jobids; bool native_launch; + size_t evindex; + opal_list_t single_events; + opal_list_t multi_events; + opal_list_t default_events; + int cache_size; + opal_list_t cache; } mca_pmix_ext20_component_t; OPAL_DECLSPEC extern mca_pmix_ext20_component_t mca_pmix_ext20_component; @@ -49,11 +55,52 @@ typedef struct { opal_list_item_t super; opal_jobid_t jobid; char nspace[PMIX_MAX_NSLEN + 1]; -} opal_pmix1_jobid_trkr_t; -OBJ_CLASS_DECLARATION(opal_pmix1_jobid_trkr_t); +} opal_pmix20_jobid_trkr_t; +OBJ_CLASS_DECLARATION(opal_pmix20_jobid_trkr_t); + +typedef struct { + opal_list_item_t super; + size_t index; + int code; + opal_pmix_notification_fn_t handler; +} opal_pmix20_single_event_t; +OBJ_CLASS_DECLARATION(opal_pmix20_single_event_t); + +typedef struct { + opal_list_item_t super; + size_t index; + int *codes; + size_t ncodes; + opal_pmix_notification_fn_t handler; +} opal_pmix20_multi_event_t; +OBJ_CLASS_DECLARATION(opal_pmix20_multi_event_t); + +typedef struct { + opal_list_item_t super; + size_t index; + opal_pmix_notification_fn_t handler; +} opal_pmix20_default_event_t; +OBJ_CLASS_DECLARATION(opal_pmix20_default_event_t); + +typedef struct { + opal_list_item_t super; + int status; + bool nondefault; + opal_process_name_t source; + pmix_data_range_t range; + opal_list_t *info; + opal_list_t results; + opal_pmix20_single_event_t *sing; + opal_pmix20_multi_event_t *multi; + opal_pmix20_default_event_t *def; + opal_pmix_op_cbfunc_t final_cbfunc; + void *final_cbdata; +} opal_pmix20_event_chain_t; +OBJ_CLASS_DECLARATION(opal_pmix20_event_chain_t); typedef struct { opal_object_t super; + pmix_status_t status; pmix_proc_t p; pmix_proc_t *procs; size_t nprocs; @@ -63,14 +110,15 @@ typedef struct { size_t ninfo; pmix_app_t *apps; size_t sz; + volatile bool active; opal_pmix_op_cbfunc_t opcbfunc; opal_pmix_modex_cbfunc_t mdxcbfunc; opal_pmix_value_cbfunc_t valcbfunc; opal_pmix_lookup_cbfunc_t lkcbfunc; opal_pmix_spawn_cbfunc_t spcbfunc; void *cbdata; -} pmix1_opcaddy_t; -OBJ_CLASS_DECLARATION(pmix1_opcaddy_t); +} pmix20_opcaddy_t; +OBJ_CLASS_DECLARATION(pmix20_opcaddy_t); typedef struct { opal_object_t super; @@ -85,90 +133,176 @@ typedef struct { void *cbdata; opal_pmix_release_cbfunc_t odmdxfunc; void *ocbdata; -} pmix1_opalcaddy_t; -OBJ_CLASS_DECLARATION(pmix1_opalcaddy_t); +} pmix20_opalcaddy_t; +OBJ_CLASS_DECLARATION(pmix20_opalcaddy_t); +typedef struct { + opal_object_t super; + opal_event_t ev; + volatile bool active; + size_t id; + int status; + opal_process_name_t pname; + opal_jobid_t jobid; + const opal_process_name_t *source; + opal_pmix_data_range_t range; + bool nondefault; + size_t handler; + opal_list_t *event_codes; + opal_list_t *info; + opal_pmix_notification_fn_t evhandler; + opal_pmix_evhandler_reg_cbfunc_t cbfunc; + opal_pmix_op_cbfunc_t opcbfunc; + void *cbdata; +} pmix20_threadshift_t; +OBJ_CLASS_DECLARATION(pmix20_threadshift_t); + +#define OPAL_PMIX_OPCD_THREADSHIFT(i, s, sr, if, nif, fn, cb, cd) \ + do { \ + pmix20_opalcaddy_t *_cd; \ + _cd = OBJ_NEW(pmix20_opalcaddy_t); \ + _cd->id = (i); \ + _cd->status = (s); \ + _cd->source = (sr); \ + _cd->info = (i); \ + _cd->evcbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_OP_THREADSHIFT(e, fn, cb, cd) \ + do { \ + pmix20_threadshift_t *_cd; \ + _cd = OBJ_NEW(pmix20_threadshift_t); \ + _cd->handler = (e); \ + _cd->opcbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_THREADSHIFT(e, i, eh, fn, cb, cd) \ + do { \ + pmix20_threadshift_t *_cd; \ + _cd = OBJ_NEW(pmix20_threadshift_t); \ + _cd->event_codes = (e); \ + _cd->info = (i); \ + _cd->evhandler = (eh); \ + _cd->cbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_NOTIFY_THREADSHIFT(s, sr, r, i, fn, cb, cd) \ + do { \ + pmix20_threadshift_t *_cd; \ + _cd = OBJ_NEW(pmix20_threadshift_t); \ + _cd->status = (s); \ + _cd->source = (sr); \ + _cd->range = (r); \ + _cd->info = (i); \ + _cd->opcbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) /**** CLIENT FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC int pmix1_client_init(void); -OPAL_MODULE_DECLSPEC int pmix1_client_finalize(void); -OPAL_MODULE_DECLSPEC int pmix1_initialized(void); -OPAL_MODULE_DECLSPEC int pmix1_abort(int flag, const char *msg, +OPAL_MODULE_DECLSPEC int pmix20_client_init(void); +OPAL_MODULE_DECLSPEC int pmix20_client_finalize(void); +OPAL_MODULE_DECLSPEC int pmix20_initialized(void); +OPAL_MODULE_DECLSPEC int pmix20_abort(int flag, const char *msg, opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_commit(void); -OPAL_MODULE_DECLSPEC int pmix1_fence(opal_list_t *procs, int collect_data); -OPAL_MODULE_DECLSPEC int pmix1_fencenb(opal_list_t *procs, int collect_data, +OPAL_MODULE_DECLSPEC int pmix20_commit(void); +OPAL_MODULE_DECLSPEC int pmix20_fence(opal_list_t *procs, int collect_data); +OPAL_MODULE_DECLSPEC int pmix20_fencenb(opal_list_t *procs, int collect_data, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_put(opal_pmix_scope_t scope, +OPAL_MODULE_DECLSPEC int pmix20_put(opal_pmix_scope_t scope, opal_value_t *val); -OPAL_MODULE_DECLSPEC int pmix1_get(const opal_process_name_t *proc, const char *key, +OPAL_MODULE_DECLSPEC int pmix20_get(const opal_process_name_t *proc, const char *key, opal_list_t *info, opal_value_t **val); -OPAL_MODULE_DECLSPEC int pmix1_getnb(const opal_process_name_t *proc, const char *key, +OPAL_MODULE_DECLSPEC int pmix20_getnb(const opal_process_name_t *proc, const char *key, opal_list_t *info, opal_pmix_value_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_publish(opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_publishnb(opal_list_t *info, +OPAL_MODULE_DECLSPEC int pmix20_publish(opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix20_publishnb(opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_lookup(opal_list_t *data, opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_lookupnb(char **keys, opal_list_t *info, +OPAL_MODULE_DECLSPEC int pmix20_lookup(opal_list_t *data, opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix20_lookupnb(char **keys, opal_list_t *info, opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_unpublish(char **keys, opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_unpublishnb(char **keys, opal_list_t *info, +OPAL_MODULE_DECLSPEC int pmix20_unpublish(char **keys, opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix20_unpublishnb(char **keys, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid); -OPAL_MODULE_DECLSPEC int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, +OPAL_MODULE_DECLSPEC int pmix20_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid); +OPAL_MODULE_DECLSPEC int pmix20_spawnnb(opal_list_t *job_info, opal_list_t *apps, opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_connect(opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_connectnb(opal_list_t *procs, +OPAL_MODULE_DECLSPEC int pmix20_connect(opal_list_t *procs); +OPAL_MODULE_DECLSPEC int pmix20_connectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_disconnect(opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_disconnectnb(opal_list_t *procs, +OPAL_MODULE_DECLSPEC int pmix20_disconnect(opal_list_t *procs); +OPAL_MODULE_DECLSPEC int pmix20_disconnectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, +OPAL_MODULE_DECLSPEC int pmix20_resolve_peers(const char *nodename, opal_jobid_t jobid, opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_resolve_nodes(opal_jobid_t jobid, char **nodelist); +OPAL_MODULE_DECLSPEC int pmix20_resolve_nodes(opal_jobid_t jobid, char **nodelist); /**** COMMON FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC int pmix1_store_local(const opal_process_name_t *proc, +OPAL_MODULE_DECLSPEC int pmix20_store_local(const opal_process_name_t *proc, opal_value_t *val); /**** SERVER SOUTHBOUND FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC int pmix1_server_init(opal_pmix_server_module_t *module, +OPAL_MODULE_DECLSPEC int pmix20_server_init(opal_pmix_server_module_t *module, opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_server_finalize(void); -OPAL_MODULE_DECLSPEC int pmix1_server_gen_regex(const char *input, char **regex); -OPAL_MODULE_DECLSPEC int pmix1_server_gen_ppn(const char *input, char **ppn); -OPAL_MODULE_DECLSPEC int pmix1_server_register_nspace(opal_jobid_t jobid, +OPAL_MODULE_DECLSPEC int pmix20_server_finalize(void); +OPAL_MODULE_DECLSPEC int pmix20_server_gen_regex(const char *input, char **regex); +OPAL_MODULE_DECLSPEC int pmix20_server_gen_ppn(const char *input, char **ppn); +OPAL_MODULE_DECLSPEC int pmix20_server_register_nspace(opal_jobid_t jobid, int nlocalprocs, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC void pmix1_server_deregister_nspace(opal_jobid_t jobid); -OPAL_MODULE_DECLSPEC int pmix1_server_register_client(const opal_process_name_t *proc, +OPAL_MODULE_DECLSPEC void pmix20_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int pmix20_server_register_client(const opal_process_name_t *proc, uid_t uid, gid_t gid, void *server_object, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC void pmix1_server_deregister_client(const opal_process_name_t *proc); -OPAL_MODULE_DECLSPEC int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env); -OPAL_MODULE_DECLSPEC int pmix1_server_dmodex(const opal_process_name_t *proc, +OPAL_MODULE_DECLSPEC void pmix20_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int pmix20_server_setup_fork(const opal_process_name_t *proc, char ***env); +OPAL_MODULE_DECLSPEC int pmix20_server_dmodex(const opal_process_name_t *proc, opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_server_notify_error(int status, - opal_list_t *procs, - opal_list_t *error_procs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix20_server_notify_event(int status, + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); /**** COMPONENT UTILITY FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC pmix_status_t pmix1_convert_opalrc(int rc); -OPAL_MODULE_DECLSPEC int pmix1_convert_rc(pmix_status_t rc); -OPAL_MODULE_DECLSPEC pmix_scope_t pmix1_convert_opalscope(opal_pmix_scope_t scope); -OPAL_MODULE_DECLSPEC void pmix1_value_load(pmix_value_t *v, +OPAL_MODULE_DECLSPEC void pmix20_event_hdlr(size_t evhdlr_registration_id, + pmix_status_t status, const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC pmix_status_t pmix20_convert_opalrc(int rc); +OPAL_MODULE_DECLSPEC int pmix20_convert_rc(pmix_status_t rc); +OPAL_MODULE_DECLSPEC pmix_scope_t pmix20_convert_opalscope(opal_pmix_scope_t scope); +OPAL_MODULE_DECLSPEC pmix_data_range_t pmix20_convert_opalrange(opal_pmix_data_range_t range); +OPAL_MODULE_DECLSPEC opal_pmix_data_range_t pmix20_convert_range(pmix_data_range_t range); +OPAL_MODULE_DECLSPEC void pmix20_value_load(pmix_value_t *v, opal_value_t *kv); -OPAL_MODULE_DECLSPEC int pmix1_value_unload(opal_value_t *kv, +OPAL_MODULE_DECLSPEC int pmix20_value_unload(opal_value_t *kv, const pmix_value_t *v); END_C_DECLS diff --git a/opal/mca/pmix/ext20/pmix_ext20_client.c b/opal/mca/pmix/ext20/pmix_ext20_client.c index 14d2b97b23..2f35e9a401 100644 --- a/opal/mca/pmix/ext20/pmix_ext20_client.c +++ b/opal/mca/pmix/ext20/pmix_ext20_client.c @@ -43,55 +43,6 @@ static size_t errhdler_ref = 0; } while (0) -static void completion_handler (void *cbdata) { - bool *active = (bool *)cbdata; - *active = false; -} - -static void myerr(size_t evhdlr_registration_id, - pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo, - pmix_event_notification_cbfunc_fn_t cbfunc, - void *cbdata) -{ - int rc; - opal_list_t plist, ilist; - opal_namelist_t *nm; - opal_value_t *iptr; - size_t n; - volatile bool active; - - /* convert the incoming status */ - rc = pmix1_convert_rc(status); - - /* convert the array of procs */ - OBJ_CONSTRUCT(&plist, opal_list_t); - for (n=0; n < nprocs; n++) { - nm = OBJ_NEW(opal_namelist_t); - nm->name.jobid = strtoul(procs[n].nspace, NULL, 10); - nm->name.vpid = procs[n].rank; - opal_list_append(&plist, &nm->super); - } - - /* convert the array of info */ - OBJ_CONSTRUCT(&ilist, opal_list_t); - for (n=0; n < ninfo; n++) { - iptr = OBJ_NEW(opal_value_t); - iptr->key = strdup(info[n].key); - pmix1_value_unload(iptr, &info[n].value); - opal_list_append(&plist, &iptr->super); - } - - /* call the base errhandler */ - active = true; - opal_pmix_base_errhandler(rc, &plist, &ilist, completion_handler, (void *)&active); - PMIX_WAIT_FOR_COMPLETION(active); - - OPAL_LIST_DESTRUCT(&plist); - OPAL_LIST_DESTRUCT(&ilist); -} - static void errreg_cbfunc (pmix_status_t status, size_t errhandler_ref, void *cbdata) @@ -102,12 +53,12 @@ static void errreg_cbfunc (pmix_status_t status, status, (unsigned long)errhandler_ref); } -int pmix1_client_init(void) +int pmix20_client_init(void) { opal_process_name_t pname; pmix_status_t rc; int dbg; - opal_pmix1_jobid_trkr_t *job; + opal_pmix20_jobid_trkr_t *job; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client init"); @@ -119,7 +70,7 @@ int pmix1_client_init(void) rc = PMIx_Init(&my_proc, NULL, 0); if (PMIX_SUCCESS != rc) { - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } /* store our jobid and rank */ @@ -135,7 +86,7 @@ int pmix1_client_init(void) } /* insert this into our list of jobids - it will be the * first, and so we'll check it first */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, my_proc.nspace, PMIX_MAX_NSLEN); job->jobid = pname.jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); @@ -144,12 +95,12 @@ int pmix1_client_init(void) opal_proc_set_name(&pname); /* register the default event handler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, myerr, errreg_cbfunc, NULL); + PMIx_Register_event_handler(NULL, 0, NULL, 0, pmix20_event_hdlr, errreg_cbfunc, NULL); return OPAL_SUCCESS; } -int pmix1_client_finalize(void) +int pmix20_client_finalize(void) { pmix_status_t rc; @@ -160,10 +111,10 @@ int pmix1_client_finalize(void) PMIx_Deregister_event_handler(errhdler_ref, NULL, NULL); rc = PMIx_Finalize(NULL, 0); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_initialized(void) +int pmix20_initialized(void) { opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client initialized"); @@ -171,14 +122,14 @@ int pmix1_initialized(void) return PMIx_Initialized(); } -int pmix1_abort(int flag, const char *msg, +int pmix20_abort(int flag, const char *msg, opal_list_t *procs) { pmix_status_t rc; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client abort"); @@ -192,7 +143,7 @@ int pmix1_abort(int flag, const char *msg, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; @@ -213,21 +164,24 @@ int pmix1_abort(int flag, const char *msg, /* release the array */ PMIX_PROC_FREE(parray, cnt); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_store_local(const opal_process_name_t *proc, opal_value_t *val) +int pmix20_store_local(const opal_process_name_t *proc, opal_value_t *val) { pmix_value_t kv; pmix_status_t rc; pmix_proc_t p; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ if (NULL != proc) { /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == proc->jobid) { job = jptr; break; @@ -246,40 +200,40 @@ int pmix1_store_local(const opal_process_name_t *proc, opal_value_t *val) } PMIX_VALUE_CONSTRUCT(&kv); - pmix1_value_load(&kv, val); + pmix20_value_load(&kv, val); rc = PMIx_Store_internal(&p, val->key, &kv); PMIX_VALUE_DESTRUCT(&kv); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_commit(void) +int pmix20_commit(void) { pmix_status_t rc; rc = PMIx_Commit(); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } static void opcbfunc(pmix_status_t status, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix20_opcaddy_t *op = (pmix20_opcaddy_t*)cbdata; if (NULL != op->opcbfunc) { - op->opcbfunc(pmix1_convert_rc(status), op->cbdata); + op->opcbfunc(pmix20_convert_rc(status), op->cbdata); } OBJ_RELEASE(op); } -int pmix1_fence(opal_list_t *procs, int collect_data) +int pmix20_fence(opal_list_t *procs, int collect_data) { pmix_status_t rc; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; pmix_info_t info, *iptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client fence"); @@ -293,7 +247,7 @@ int pmix1_fence(opal_list_t *procs, int collect_data) /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; @@ -328,20 +282,23 @@ int pmix1_fence(opal_list_t *procs, int collect_data) PMIX_INFO_DESTRUCT(&info); } - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_fencenb(opal_list_t *procs, int collect_data, +int pmix20_fencenb(opal_list_t *procs, int collect_data, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t rc; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; pmix_info_t info, *iptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client fence_nb"); @@ -355,7 +312,7 @@ int pmix1_fencenb(opal_list_t *procs, int collect_data, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; @@ -383,7 +340,7 @@ int pmix1_fencenb(opal_list_t *procs, int collect_data, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; op->procs = parray; @@ -395,29 +352,29 @@ int pmix1_fencenb(opal_list_t *procs, int collect_data, OBJ_RELEASE(op); } - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_put(opal_pmix_scope_t opal_scope, +int pmix20_put(opal_pmix_scope_t opal_scope, opal_value_t *val) { pmix_value_t kv; - pmix_scope_t pmix_scope = pmix1_convert_opalscope(opal_scope); + pmix_scope_t pmix_scope = pmix20_convert_opalscope(opal_scope); pmix_status_t rc; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client put"); PMIX_VALUE_CONSTRUCT(&kv); - pmix1_value_load(&kv, val); + pmix20_value_load(&kv, val); rc = PMIx_Put(pmix_scope, val->key, &kv); PMIX_VALUE_DESTRUCT(&kv); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_get(const opal_process_name_t *proc, const char *key, +int pmix20_get(const opal_process_name_t *proc, const char *key, opal_list_t *info, opal_value_t **val) { int ret; @@ -427,7 +384,7 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, size_t ninfo, n; pmix_info_t *pinfo; opal_value_t *ival; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "%s PMIx_client get on proc %s key %s", @@ -440,7 +397,7 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == proc->jobid) { job = jptr; break; @@ -475,7 +432,7 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, n=0; OPAL_LIST_FOREACH(ival, info, opal_value_t) { (void)strncpy(pinfo[n].key, ival->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, ival); + pmix20_value_load(&pinfo[n].value, ival); } } else { pinfo = NULL; @@ -492,11 +449,11 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, ret = OPAL_SUCCESS; } else { *val = OBJ_NEW(opal_value_t); - ret = pmix1_value_unload(*val, kv); + ret = pmix20_value_unload(*val, kv); PMIX_VALUE_FREE(kv, 1); } } else { - ret = pmix1_convert_rc(rc); + ret = pmix20_convert_rc(rc); } PMIX_INFO_FREE(pinfo, ninfo); return ret; @@ -505,13 +462,13 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, static void val_cbfunc(pmix_status_t status, pmix_value_t *kv, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix20_opcaddy_t *op = (pmix20_opcaddy_t*)cbdata; int rc; opal_value_t val, *v=NULL; - rc = pmix1_convert_opalrc(status); + rc = pmix20_convert_opalrc(status); if (PMIX_SUCCESS == status && NULL != kv) { - rc = pmix1_value_unload(&val, kv); + rc = pmix20_value_unload(&val, kv); v = &val; } @@ -521,15 +478,18 @@ static void val_cbfunc(pmix_status_t status, OBJ_RELEASE(op); } -int pmix1_getnb(const opal_process_name_t *proc, const char *key, +int pmix20_getnb(const opal_process_name_t *proc, const char *key, opal_list_t *info, opal_pmix_value_cbfunc_t cbfunc, void *cbdata) { - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; pmix_status_t rc; size_t n; opal_value_t *ival; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access shared lists/objects */ opal_output_verbose(1, opal_pmix_base_framework.framework_output, "%s PMIx_client get_nb on proc %s key %s", @@ -537,7 +497,7 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, (NULL == proc) ? "NULL" : OPAL_NAME_PRINT(*proc), key); /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->valcbfunc = cbfunc; op->cbdata = cbdata; @@ -545,7 +505,7 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == proc->jobid) { job = jptr; break; @@ -568,7 +528,7 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, n=0; OPAL_LIST_FOREACH(ival, info, opal_value_t) { (void)strncpy(op->info[n].key, ival->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, ival); + pmix20_value_load(&op->info[n].value, ival); } } } @@ -579,10 +539,10 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, OBJ_RELEASE(op); } - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_publish(opal_list_t *info) +int pmix20_publish(opal_list_t *info) { pmix_info_t *pinfo; pmix_status_t ret; @@ -602,7 +562,7 @@ int pmix1_publish(opal_list_t *info) n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, iptr); + pmix20_value_load(&pinfo[n].value, iptr); ++n; } } else { @@ -611,16 +571,16 @@ int pmix1_publish(opal_list_t *info) ret = PMIx_Publish(pinfo, sz); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_publishnb(opal_list_t *info, +int pmix20_publishnb(opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; opal_value_t *iptr; size_t n; - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client publish_nb"); @@ -630,7 +590,7 @@ int pmix1_publishnb(opal_list_t *info, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; @@ -640,17 +600,17 @@ int pmix1_publishnb(opal_list_t *info, n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, iptr); + pmix20_value_load(&op->info[n].value, iptr); ++n; } } ret = PMIx_Publish_nb(op->info, op->sz, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_lookup(opal_list_t *data, opal_list_t *info) +int pmix20_lookup(opal_list_t *data, opal_list_t *info) { pmix_pdata_t *pdata; pmix_info_t *pinfo; @@ -659,8 +619,10 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) pmix_status_t ret; opal_pmix_pdata_t *d; opal_value_t *iptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; + /* we must threadshift this request as we might not be in an event + * and we are going to access shared lists/objects */ opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client lookup"); @@ -681,7 +643,7 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(pinfo[n++].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, iptr); + pmix20_value_load(&pinfo[n].value, iptr); ++n; } } else { @@ -707,14 +669,14 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) } /* if we don't already have it, add this to our jobid tracker */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == d->proc.jobid) { job = jptr; break; } } if (NULL == job) { - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, pdata[n].proc.nspace, PMIX_MAX_NSLEN); job->jobid = d->proc.jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); @@ -724,7 +686,7 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) } else { d->proc.vpid = pdata[n].proc.rank; } - rc = pmix1_value_unload(&d->value, &pdata[n].value); + rc = pmix20_value_unload(&d->value, &pdata[n].value); if (OPAL_SUCCESS != rc) { OPAL_ERROR_LOG(rc); PMIX_PDATA_FREE(pdata, sz); @@ -734,26 +696,30 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) } } - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } static void lk_cbfunc(pmix_status_t status, pmix_pdata_t data[], size_t ndata, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix20_opcaddy_t *op = (pmix20_opcaddy_t*)cbdata; opal_pmix_pdata_t *d; opal_list_t results, *r = NULL; int rc; size_t n; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; - if (NULL == op->lkcbfunc) { + /* this is in the PMIx local thread - need to threadshift to + * our own thread as we will be accessing framework-global + * lists and objects */ + + if (NULL == op->lkcbfunc) { OBJ_RELEASE(op); return; } - rc = pmix1_convert_rc(status); + rc = pmix20_convert_rc(status); if (OPAL_SUCCESS == rc) { OBJ_CONSTRUCT(&results, opal_list_t); for (n=0; n < ndata; n++) { @@ -770,14 +736,14 @@ static void lk_cbfunc(pmix_status_t status, } /* if we don't already have it, add this to our jobid tracker */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == d->proc.jobid) { job = jptr; break; } } if (NULL == job) { - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, data[n].proc.nspace, PMIX_MAX_NSLEN); job->jobid = d->proc.jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); @@ -788,7 +754,7 @@ static void lk_cbfunc(pmix_status_t status, d->proc.vpid = data[n].proc.rank; } d->value.key = strdup(data[n].key); - rc = pmix1_value_unload(&d->value, &data[n].value); + rc = pmix20_value_unload(&d->value, &data[n].value); if (OPAL_SUCCESS != rc) { rc = OPAL_ERR_BAD_PARAM; OPAL_ERROR_LOG(rc); @@ -807,11 +773,11 @@ static void lk_cbfunc(pmix_status_t status, OBJ_RELEASE(op); } -int pmix1_lookupnb(char **keys, opal_list_t *info, +int pmix20_lookupnb(char **keys, opal_list_t *info, opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; opal_value_t *iptr; size_t n; @@ -820,7 +786,7 @@ int pmix1_lookupnb(char **keys, opal_list_t *info, "PMIx_client lookup_nb"); /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->lkcbfunc = cbfunc; op->cbdata = cbdata; @@ -831,7 +797,7 @@ int pmix1_lookupnb(char **keys, opal_list_t *info, n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, iptr); + pmix20_value_load(&op->info[n].value, iptr); ++n; } } @@ -839,10 +805,10 @@ int pmix1_lookupnb(char **keys, opal_list_t *info, ret = PMIx_Lookup_nb(keys, op->info, op->sz, lk_cbfunc, op); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_unpublish(char **keys, opal_list_t *info) +int pmix20_unpublish(char **keys, opal_list_t *info) { pmix_status_t ret; size_t ninfo, n; @@ -855,7 +821,7 @@ int pmix1_unpublish(char **keys, opal_list_t *info) n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(pinfo[n++].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, iptr); + pmix20_value_load(&pinfo[n].value, iptr); ++n; } } else { @@ -866,19 +832,19 @@ int pmix1_unpublish(char **keys, opal_list_t *info) ret = PMIx_Unpublish(keys, pinfo, ninfo); PMIX_INFO_FREE(pinfo, ninfo); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_unpublishnb(char **keys, opal_list_t *info, +int pmix20_unpublishnb(char **keys, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; opal_value_t *iptr; size_t n; /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; @@ -889,7 +855,7 @@ int pmix1_unpublishnb(char **keys, opal_list_t *info, n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, iptr); + pmix20_value_load(&op->info[n].value, iptr); ++n; } } @@ -897,10 +863,10 @@ int pmix1_unpublishnb(char **keys, opal_list_t *info, ret = PMIx_Unpublish_nb(keys, op->info, op->sz, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) +int pmix20_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) { pmix_status_t ret; pmix_info_t *pinfo = NULL; @@ -909,14 +875,14 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) char nspace[PMIX_MAX_NSLEN+1]; opal_value_t *info; opal_pmix_app_t *app; - opal_pmix1_jobid_trkr_t *job; + opal_pmix20_jobid_trkr_t *job; if (NULL != job_info && 0 < (ninfo = opal_list_get_size(job_info))) { PMIX_INFO_CREATE(pinfo, ninfo); n=0; OPAL_LIST_FOREACH(info, job_info, opal_value_t) { (void)strncpy(pinfo[n].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, info); + pmix20_value_load(&pinfo[n].value, info); ++n; } } @@ -935,7 +901,7 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) m=0; OPAL_LIST_FOREACH(info, &app->info, opal_value_t) { (void)strncpy(papps[n].info[m].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&papps[n].info[m].value, info); + pmix20_value_load(&papps[n].info[m].value, info); ++m; } } @@ -954,25 +920,29 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) OPAL_HASH_STR(nspace, *jobid); } /* add this to our jobid tracker */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); job->jobid = *jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); } PMIX_APP_FREE(papps, napps); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } static void spcbfunc(pmix_status_t status, char *nspace, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix20_opcaddy_t *op = (pmix20_opcaddy_t*)cbdata; int rc; opal_jobid_t jobid=OPAL_JOBID_INVALID; - opal_pmix1_jobid_trkr_t *job; + opal_pmix20_jobid_trkr_t *job; - rc = pmix1_convert_rc(status); + /* this is in the PMIx local thread - need to threadshift to + * our own thread as we will be accessing framework-global + * lists and objects */ + + rc = pmix20_convert_rc(status); if (PMIX_SUCCESS == status) { if (mca_pmix_ext20_component.native_launch) { /* if we were launched by the OMPI RTE, then @@ -984,7 +954,7 @@ static void spcbfunc(pmix_status_t status, OPAL_HASH_STR(nspace, jobid); } /* add this to our jobid tracker */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); job->jobid = jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); @@ -994,17 +964,17 @@ static void spcbfunc(pmix_status_t status, OBJ_RELEASE(op); } -int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, +int pmix20_spawnnb(opal_list_t *job_info, opal_list_t *apps, opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; size_t n, m; opal_value_t *info; opal_pmix_app_t *app; /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->spcbfunc = cbfunc; op->cbdata = cbdata; @@ -1013,7 +983,7 @@ int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, n=0; OPAL_LIST_FOREACH(info, job_info, opal_value_t) { (void)strncpy(op->info[n].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, info); + pmix20_value_load(&op->info[n].value, info); ++n; } } @@ -1032,7 +1002,7 @@ int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, m=0; OPAL_LIST_FOREACH(info, &app->info, opal_value_t) { (void)strncpy(op->apps[n].info[m].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->apps[n].info[m].value, info); + pmix20_value_load(&op->apps[n].info[m].value, info); ++m; } } @@ -1041,16 +1011,16 @@ int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, ret = PMIx_Spawn_nb(op->info, op->ninfo, op->apps, op->sz, spcbfunc, op); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_connect(opal_list_t *procs) +int pmix20_connect(opal_list_t *procs) { pmix_status_t ret; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1065,7 +1035,7 @@ int pmix1_connect(opal_list_t *procs) /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; @@ -1087,18 +1057,21 @@ int pmix1_connect(opal_list_t *procs) ret = PMIx_Connect(parray, cnt, NULL, 0); PMIX_PROC_FREE(parray, cnt); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_connectnb(opal_list_t *procs, +int pmix20_connectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; size_t n, cnt=0; opal_namelist_t *ptr; - pmix1_opcaddy_t *op; - opal_pmix1_jobid_trkr_t *job; + pmix20_opcaddy_t *op; + opal_pmix20_jobid_trkr_t *job; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1106,7 +1079,7 @@ int pmix1_connectnb(opal_list_t *procs, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; op->nprocs = cnt; @@ -1118,7 +1091,7 @@ int pmix1_connectnb(opal_list_t *procs, OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { /* look thru our list of jobids and find the * corresponding nspace */ - OPAL_LIST_FOREACH(job, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(job, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (job->jobid == ptr->name.jobid) { (void)strncpy(op->procs[n].nspace, job->nspace, PMIX_MAX_NSLEN); break; @@ -1134,16 +1107,16 @@ int pmix1_connectnb(opal_list_t *procs, ret = PMIx_Connect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_disconnect(opal_list_t *procs) +int pmix20_disconnect(opal_list_t *procs) { pmix_status_t ret; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - opal_pmix1_jobid_trkr_t *job; + opal_pmix20_jobid_trkr_t *job; /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1157,7 +1130,7 @@ int pmix1_disconnect(opal_list_t *procs) OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { /* look thru our list of jobids and find the * corresponding nspace */ - OPAL_LIST_FOREACH(job, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(job, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (job->jobid == ptr->name.jobid) { (void)strncpy(parray[n].nspace, job->nspace, PMIX_MAX_NSLEN); break; @@ -1174,18 +1147,21 @@ int pmix1_disconnect(opal_list_t *procs) ret = PMIx_Disconnect(parray, cnt, NULL, 0); PMIX_PROC_FREE(parray, cnt); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_disconnectnb(opal_list_t *procs, +int pmix20_disconnectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; size_t n, cnt=0; opal_namelist_t *ptr; - pmix1_opcaddy_t *op; - opal_pmix1_jobid_trkr_t *job; + pmix20_opcaddy_t *op; + opal_pmix20_jobid_trkr_t *job; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1193,7 +1169,7 @@ int pmix1_disconnectnb(opal_list_t *procs, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; op->nprocs = cnt; @@ -1205,7 +1181,7 @@ int pmix1_disconnectnb(opal_list_t *procs, OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { /* look thru our list of jobids and find the * corresponding nspace */ - OPAL_LIST_FOREACH(job, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(job, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (job->jobid == ptr->name.jobid) { (void)strncpy(op->procs[n].nspace, job->nspace, PMIX_MAX_NSLEN); break; @@ -1221,11 +1197,11 @@ int pmix1_disconnectnb(opal_list_t *procs, ret = PMIx_Disconnect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix20_convert_rc(ret); } -int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, +int pmix20_resolve_peers(const char *nodename, opal_jobid_t jobid, opal_list_t *procs) { char *nspace; @@ -1234,13 +1210,16 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, opal_namelist_t *nm; int rc; pmix_status_t ret; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ if (OPAL_JOBID_WILDCARD == jobid) { nspace = NULL; } else { job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == jobid) { job = jptr; break; @@ -1253,7 +1232,7 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, } ret = PMIx_Resolve_peers(nodename, nspace, &array, &nprocs); - rc = pmix1_convert_rc(ret); + rc = pmix20_convert_rc(ret); if (NULL != array && 0 < nprocs) { for (n=0; n < nprocs; n++) { @@ -1270,14 +1249,14 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, } /* if we don't already have it, add this to our jobid tracker */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == nm->name.jobid) { job = jptr; break; } } if (NULL == job) { - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); job->jobid = jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); @@ -1290,17 +1269,20 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, return rc; } -int pmix1_resolve_nodes(opal_jobid_t jobid, char **nodelist) +int pmix20_resolve_nodes(opal_jobid_t jobid, char **nodelist) { pmix_status_t ret; char *nspace=NULL; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix20_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ if (OPAL_JOBID_WILDCARD != jobid) { /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { if (jptr->jobid == jobid) { job = jptr; break; @@ -1314,5 +1296,5 @@ int pmix1_resolve_nodes(opal_jobid_t jobid, char **nodelist) ret = PMIx_Resolve_nodes(nspace, nodelist); - return pmix1_convert_rc(ret);; + return pmix20_convert_rc(ret);; } diff --git a/opal/mca/pmix/ext20/pmix_ext20_component.c b/opal/mca/pmix/ext20/pmix_ext20_component.c index 52377e549a..0c71cd1e89 100644 --- a/opal/mca/pmix/ext20/pmix_ext20_component.c +++ b/opal/mca/pmix/ext20/pmix_ext20_component.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -35,6 +35,7 @@ const char *opal_pmix_ext20_component_version_string = static int external_open(void); static int external_close(void); static int external_component_query(mca_base_module_t **module, int *priority); +static int external_register(void); /* @@ -64,6 +65,7 @@ mca_pmix_ext20_component_t mca_pmix_ext20_component = { .mca_open_component = external_open, .mca_close_component = external_close, .mca_query_component = external_component_query, + .mca_register_component_params = external_register, }, /* Next the MCA v1.0.0 component meta data */ .base_data = { @@ -74,15 +76,38 @@ mca_pmix_ext20_component_t mca_pmix_ext20_component = { .native_launch = false }; +static int external_register(void) +{ + mca_pmix_ext20_component.cache_size = 256; + mca_base_component_var_register(&mca_pmix_ext20_component.super.base_version, + "cache_size", "Size of the ring buffer cache for events", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, + MCA_BASE_VAR_SCOPE_CONSTANT, + &mca_pmix_ext20_component.cache_size); + + return OPAL_SUCCESS; +} + + static int external_open(void) { + mca_pmix_ext20_component.evindex = 0; OBJ_CONSTRUCT(&mca_pmix_ext20_component.jobids, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_ext20_component.single_events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_ext20_component.multi_events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_ext20_component.default_events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_ext20_component.cache, opal_list_t); + return OPAL_SUCCESS; } static int external_close(void) { OPAL_LIST_DESTRUCT(&mca_pmix_ext20_component.jobids); + OPAL_LIST_DESTRUCT(&mca_pmix_ext20_component.single_events); + OPAL_LIST_DESTRUCT(&mca_pmix_ext20_component.multi_events); + OPAL_LIST_DESTRUCT(&mca_pmix_ext20_component.default_events); + OPAL_LIST_DESTRUCT(&mca_pmix_ext20_component.cache); return OPAL_SUCCESS; } diff --git a/opal/mca/pmix/ext20/pmix_ext20_server_north.c b/opal/mca/pmix/ext20/pmix_ext20_server_north.c index 705f594f6b..fe319e7a53 100644 --- a/opal/mca/pmix/ext20/pmix_ext20_server_north.c +++ b/opal/mca/pmix/ext20/pmix_ext20_server_north.c @@ -44,7 +44,8 @@ /* These are the interfaces used by the embedded PMIx server * to call up into ORTE for service requests */ - static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object); + static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object, @@ -83,13 +84,11 @@ static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t server_notify_event(pmix_status_t code, - pmix_proc_t procs[], size_t nprocs, + const pmix_proc_t *source, + pmix_data_range_t range, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); - static pmix_status_t server_listener_fn(int listening_sd, - pmix_connection_cbfunc_t cbfunc); - - pmix_server_module_t mymodule = { + pmix_server_module_t mymodule = { .client_connected = server_client_connected_fn, .client_finalized = server_client_finalized_fn, .abort = server_abort_fn, @@ -103,8 +102,7 @@ .disconnect = server_disconnect_fn, .register_events = server_register_events, .deregister_events = server_deregister_events, - .notify_event = server_notify_event, - .listener = server_listener_fn + .notify_event = server_notify_event }; opal_pmix_server_module_t *host_module = NULL; @@ -112,39 +110,46 @@ opal_pmix_server_module_t *host_module = NULL; static void opal_opcbfunc(int status, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix20_opalcaddy_t *opalcaddy = (pmix20_opalcaddy_t*)cbdata; if (NULL != opalcaddy->opcbfunc) { - opalcaddy->opcbfunc(pmix1_convert_opalrc(status), opalcaddy->cbdata); + opalcaddy->opcbfunc(pmix20_convert_opalrc(status), opalcaddy->cbdata); } OBJ_RELEASE(opalcaddy); } -static pmix_status_t server_client_connected_fn(const pmix_proc_t *p, void *server_object) +static pmix_status_t server_client_connected_fn(const pmix_proc_t *p, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; opal_process_name_t proc; + pmix20_opalcaddy_t *opalcaddy; if (NULL == host_module || NULL == host_module->client_connected) { return PMIX_SUCCESS; } + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } proc.vpid = p->rank; /* pass it up */ - rc = host_module->client_connected(&proc, server_object); - return pmix1_convert_opalrc(rc); + rc = host_module->client_connected(&proc, server_object, + opal_opcbfunc, opalcaddy); + return pmix20_convert_opalrc(rc); } static pmix_status_t server_client_finalized_fn(const pmix_proc_t *p, void* server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_process_name_t proc; if (NULL == host_module || NULL == host_module->client_finalized) { @@ -153,12 +158,12 @@ static pmix_status_t server_client_finalized_fn(const pmix_proc_t *p, void* serv /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } proc.vpid = p->rank; /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -167,7 +172,7 @@ static pmix_status_t server_client_finalized_fn(const pmix_proc_t *p, void* serv if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, @@ -179,7 +184,7 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, opal_namelist_t *nm; opal_process_name_t proc; int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; if (NULL == host_module || NULL == host_module->abort) { return PMIX_ERR_NOT_SUPPORTED; @@ -187,12 +192,12 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } proc.vpid = p->rank; /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -202,7 +207,7 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -217,12 +222,12 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static void _data_release(void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix20_opalcaddy_t *opalcaddy = (pmix20_opalcaddy_t*)cbdata; if (NULL != opalcaddy->odmdxfunc) { opalcaddy->odmdxfunc(opalcaddy->ocbdata); @@ -234,9 +239,9 @@ static void opmdx_response(int status, const char *data, size_t sz, void *cbdata opal_pmix_release_cbfunc_t relcbfunc, void *relcbdata) { pmix_status_t rc; - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix20_opalcaddy_t *opalcaddy = (pmix20_opalcaddy_t*)cbdata; - rc = pmix1_convert_rc(status); + rc = pmix20_convert_rc(status); if (NULL != opalcaddy->mdxcbfunc) { opalcaddy->odmdxfunc = relcbfunc; opalcaddy->ocbdata = relcbdata; @@ -252,7 +257,7 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; size_t n; opal_namelist_t *nm; opal_value_t *iptr; @@ -263,7 +268,7 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->mdxcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -273,7 +278,7 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -287,9 +292,9 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -299,7 +304,7 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, @@ -307,7 +312,7 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, pmix_modex_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *iptr; size_t n; @@ -318,7 +323,7 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -327,7 +332,7 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->mdxcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -336,9 +341,9 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -350,7 +355,7 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, if (OPAL_ERR_IN_PROCESS == rc) { rc = OPAL_SUCCESS; } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static pmix_status_t server_publish_fn(const pmix_proc_t *p, @@ -359,7 +364,7 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, { int rc; size_t n; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *oinfo; @@ -369,7 +374,7 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -378,7 +383,7 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -387,9 +392,9 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -399,21 +404,21 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static void opal_lkupcbfunc(int status, opal_list_t *data, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix20_opalcaddy_t *opalcaddy = (pmix20_opalcaddy_t*)cbdata; pmix_status_t rc; pmix_pdata_t *d=NULL; size_t nd=0, n; opal_pmix_pdata_t *p; if (NULL != opalcaddy->lkupcbfunc) { - rc = pmix1_convert_opalrc(status); + rc = pmix20_convert_opalrc(status); /* convert any returned data */ if (NULL != data) { nd = opal_list_get_size(data); @@ -424,7 +429,7 @@ static void opal_lkupcbfunc(int status, (void)opal_snprintf_jobid(d[n].proc.nspace, PMIX_MAX_NSLEN, p->proc.jobid); d[n].proc.rank = p->proc.vpid; (void)strncpy(d[n].key, p->value.key, PMIX_MAX_KEYLEN); - pmix1_value_load(&d[n].value, &p->value); + pmix20_value_load(&d[n].value, &p->value); } } opalcaddy->lkupcbfunc(rc, d, nd, opalcaddy->cbdata); @@ -437,7 +442,7 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, pmix_lookup_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *iptr; size_t n; @@ -448,7 +453,7 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -457,7 +462,7 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->lkupcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -466,9 +471,9 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -478,7 +483,7 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } @@ -487,7 +492,7 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *iptr; size_t n; @@ -498,7 +503,7 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -507,7 +512,7 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -516,9 +521,9 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -528,17 +533,17 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static void opal_spncbfunc(int status, opal_jobid_t jobid, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix20_opalcaddy_t *opalcaddy = (pmix20_opalcaddy_t*)cbdata; pmix_status_t rc; char nspace[PMIX_MAX_NSLEN]; if (NULL != opalcaddy->spwncbfunc) { - rc = pmix1_convert_opalrc(status); + rc = pmix20_convert_opalrc(status); /* convert the jobid */ (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid); opalcaddy->spwncbfunc(rc, nspace, opalcaddy->cbdata); @@ -551,7 +556,7 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, const pmix_app_t apps[], size_t napps, pmix_spawn_cbfunc_t cbfunc, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_pmix_app_t *app; opal_value_t *oinfo; @@ -564,7 +569,7 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -573,7 +578,7 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->spwncbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -582,9 +587,9 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(job_info[k].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &job_info[k].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(oinfo, &job_info[k].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -607,9 +612,9 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&app->info, &oinfo->super); oinfo->key = strdup(apps[n].info[k].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &apps[n].info[k].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(oinfo, &apps[n].info[k].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } } @@ -621,7 +626,7 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } @@ -630,7 +635,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_namelist_t *nm; size_t n; opal_value_t *oinfo; @@ -640,7 +645,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -650,7 +655,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -664,9 +669,9 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -676,7 +681,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } @@ -685,7 +690,7 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; opal_namelist_t *nm; size_t n; opal_value_t *oinfo; @@ -695,7 +700,7 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -705,7 +710,7 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -719,9 +724,9 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -731,20 +736,20 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy; + pmix20_opalcaddy_t *opalcaddy; size_t n; opal_value_t *oinfo; int rc; /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix20_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -753,9 +758,9 @@ static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix20_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } } @@ -765,7 +770,7 @@ static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix20_convert_opalrc(rc); } static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes, @@ -775,22 +780,10 @@ static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncode } static pmix_status_t server_notify_event(pmix_status_t code, - pmix_proc_t procs[], size_t nprocs, + const pmix_proc_t *source, + pmix_data_range_t range, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { return PMIX_ERR_NOT_SUPPORTED; } - -static pmix_status_t server_listener_fn(int listening_sd, - pmix_connection_cbfunc_t cbfunc) -{ - int rc; - - if (NULL == host_module || NULL == host_module->listener) { - return PMIX_ERR_NOT_SUPPORTED; - } - - rc = host_module->listener(listening_sd, cbfunc); - return pmix1_convert_opalrc(rc); -} diff --git a/opal/mca/pmix/ext20/pmix_ext20_server_south.c b/opal/mca/pmix/ext20/pmix_ext20_server_south.c index af37525ee1..ffb11a299c 100644 --- a/opal/mca/pmix/ext20/pmix_ext20_server_south.c +++ b/opal/mca/pmix/ext20/pmix_ext20_server_south.c @@ -3,7 +3,7 @@ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -57,66 +57,20 @@ static size_t errhdler_ref = 0; } \ } while (0) -static void completion_handler (void *cbdata) { - bool *active = (bool *)cbdata; - *active = false; -} - -static void myerr(size_t evhdlr_registration_id, - pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo, - pmix_event_notification_cbfunc_fn_t cbfunc, - void *cbdata) -{ - int rc; - opal_list_t plist, ilist; - opal_namelist_t *nm; - opal_value_t *iptr; - size_t n; - volatile bool active; - - /* convert the incoming status */ - rc = pmix1_convert_rc(status); - - /* convert the array of procs */ - OBJ_CONSTRUCT(&plist, opal_list_t); - for (n=0; n < nprocs; n++) { - nm = OBJ_NEW(opal_namelist_t); - nm->name.jobid = strtoul(procs[n].nspace, NULL, 10); - nm->name.vpid = procs[n].rank; - opal_list_append(&plist, &nm->super); - } - - /* convert the array of info */ - OBJ_CONSTRUCT(&ilist, opal_list_t); - for (n=0; n < ninfo; n++) { - iptr = OBJ_NEW(opal_value_t); - iptr->key = strdup(info[n].key); - pmix1_value_unload(iptr, &info[n].value); - opal_list_append(&plist, &iptr->super); - } - - /* call the base errhandler */ - active = true; - opal_pmix_base_errhandler(rc, &plist, &ilist, completion_handler, (void *)&active); - PMIX_WAIT_FOR_COMPLETION(active); - - OPAL_LIST_DESTRUCT(&plist); - OPAL_LIST_DESTRUCT(&ilist); -} - static void errreg_cbfunc (pmix_status_t status, size_t errhandler_ref, void *cbdata) { + volatile bool *active = (volatile bool*)cbdata; + errhdler_ref = errhandler_ref; opal_output_verbose(5, opal_pmix_base_framework.framework_output, "PMIX server errreg_cbfunc - error handler registered status=%d, reference=%lu", status, (unsigned long)errhandler_ref); + *active = false; } -int pmix1_server_init(opal_pmix_server_module_t *module, +int pmix20_server_init(opal_pmix_server_module_t *module, opal_list_t *info) { pmix_status_t rc; @@ -124,6 +78,7 @@ int pmix1_server_init(opal_pmix_server_module_t *module, opal_value_t *kv; pmix_info_t *pinfo; size_t sz, n; + volatile bool active; if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) { asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg); @@ -137,7 +92,7 @@ int pmix1_server_init(opal_pmix_server_module_t *module, n = 0; OPAL_LIST_FOREACH(kv, info, opal_value_t) { (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, kv); + pmix20_value_load(&pinfo[n].value, kv); ++n; } } else { @@ -147,7 +102,7 @@ int pmix1_server_init(opal_pmix_server_module_t *module, if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, pinfo, sz))) { PMIX_INFO_FREE(pinfo, sz); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } PMIX_INFO_FREE(pinfo, sz); @@ -155,79 +110,95 @@ int pmix1_server_init(opal_pmix_server_module_t *module, host_module = module; /* register the default event handler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, myerr, errreg_cbfunc, NULL); + active = true; + PMIx_Register_event_handler(NULL, 0, NULL, 0, pmix20_event_hdlr, errreg_cbfunc, (void*)&active); + PMIX_WAIT_FOR_COMPLETION(active); return OPAL_SUCCESS; } -int pmix1_server_finalize(void) +static void fincb(pmix_status_t status, void *cbdata) { - pmix_status_t rc; - - /* deregister the default event handler */ - PMIx_Deregister_event_handler(errhdler_ref, NULL, NULL); - - rc = PMIx_server_finalize(); - return pmix1_convert_rc(rc); + volatile bool *active = (volatile bool*)cbdata; + *active = false; } -int pmix1_server_gen_regex(const char *input, char **regex) +int pmix20_server_finalize(void) +{ + pmix_status_t rc; + volatile bool active; + + /* deregister the default event handler */ + active = true; + PMIx_Deregister_event_handler(errhdler_ref, fincb, (void*)&active); + PMIX_WAIT_FOR_COMPLETION(active); + + rc = PMIx_server_finalize(); + return pmix20_convert_rc(rc); +} + +int pmix20_server_gen_regex(const char *input, char **regex) { pmix_status_t rc; rc = PMIx_generate_regex(input, regex); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_server_gen_ppn(const char *input, char **ppn) +int pmix20_server_gen_ppn(const char *input, char **ppn) { pmix_status_t rc; rc = PMIx_generate_ppn(input, ppn); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } static void opcbfunc(pmix_status_t status, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix20_opcaddy_t *op = (pmix20_opcaddy_t*)cbdata; if (NULL != op->opcbfunc) { - op->opcbfunc(pmix1_convert_rc(status), op->cbdata); + op->opcbfunc(pmix20_convert_rc(status), op->cbdata); + } + if (op->active) { + op->status = status; + op->active = false; + } else { + OBJ_RELEASE(op); } - OBJ_RELEASE(op); } -int pmix1_server_register_nspace(opal_jobid_t jobid, - int nlocalprocs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata) +static void _reg_nspace(int sd, short args, void *cbdata) { + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; opal_value_t *kv, *k2; - pmix_info_t *pinfo, *pmap; + pmix_info_t *pinfo = NULL, *pmap; size_t sz, szmap, m, n; char nspace[PMIX_MAX_NSLEN]; pmix_status_t rc; - pmix1_opcaddy_t *op; opal_list_t *pmapinfo; - opal_pmix1_jobid_trkr_t *job; + opal_pmix20_jobid_trkr_t *job; + pmix20_opcaddy_t op; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ /* convert the jobid */ - (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid); + (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, cd->jobid); /* store this job in our list of known nspaces */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix20_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); - job->jobid = jobid; + job->jobid = cd->jobid; opal_list_append(&mca_pmix_ext20_component.jobids, &job->super); /* convert the list to an array of pmix_info_t */ - if (NULL != info) { - sz = opal_list_get_size(info); + if (NULL != cd->info) { + sz = opal_list_get_size(cd->info); PMIX_INFO_CREATE(pinfo, sz); n = 0; - OPAL_LIST_FOREACH(kv, info, opal_value_t) { + OPAL_LIST_FOREACH(kv, cd->info, opal_value_t) { (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); if (0 == strcmp(kv->key, OPAL_PMIX_PROC_DATA)) { pinfo[n].value.type = PMIX_INFO_ARRAY; @@ -241,11 +212,11 @@ int pmix1_server_register_nspace(opal_jobid_t jobid, m = 0; OPAL_LIST_FOREACH(k2, pmapinfo, opal_value_t) { (void)strncpy(pmap[m].key, k2->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pmap[m].value, k2); + pmix20_value_load(&pmap[m].value, k2); ++m; } } else { - pmix1_value_load(&pinfo[n].value, kv); + pmix20_value_load(&pinfo[n].value, kv); } ++n; } @@ -254,82 +225,185 @@ int pmix1_server_register_nspace(opal_jobid_t jobid, pinfo = NULL; } - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->info = pinfo; - op->sz = sz; - op->opcbfunc = cbfunc; - op->cbdata = cbdata; - rc = PMIx_server_register_nspace(nspace, nlocalprocs, pinfo, sz, - opcbfunc, op); - if (PMIX_SUCCESS != rc) { - OBJ_RELEASE(op); + OBJ_CONSTRUCT(&op, pmix20_opcaddy_t); + op.active = true; + rc = PMIx_server_register_nspace(nspace, cd->status, pinfo, sz, + opcbfunc, (void*)&op); + if (PMIX_SUCCESS == rc) { + PMIX_WAIT_FOR_COMPLETION(op.active); + } else { + op.status = rc; } - return pmix1_convert_rc(rc); + /* ensure we execute the cbfunc so the caller doesn't hang */ + if (NULL != cd->opcbfunc) { + cd->opcbfunc(pmix20_convert_rc(op.status), cd->cbdata); + } + if (NULL != pinfo) { + PMIX_INFO_FREE(pinfo, sz); + } + OBJ_DESTRUCT(&op); + OBJ_RELEASE(cd); } -void pmix1_server_deregister_nspace(opal_jobid_t jobid) +int pmix20_server_register_nspace(opal_jobid_t jobid, + int nlocalprocs, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) { - opal_pmix1_jobid_trkr_t *jptr; + pmix20_threadshift_t *cd; + + /* we must threadshift this request as it touches + * shared lists of objects */ + cd = OBJ_NEW(pmix20_threadshift_t); + cd->jobid = jobid; + cd->status = nlocalprocs; + cd->info = info; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + /* if the cbfunc is NULL, then the caller is in an event + * and we can directly call the processing function */ + if (NULL == cbfunc) { + _reg_nspace(0, 0, cd); + } else { + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _reg_nspace, cd); + event_active(&cd->ev, EV_WRITE, 1); + } + + return OPAL_SUCCESS; +} + +static void tdcbfunc(pmix_status_t status, void *cbdata) +{ + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + + if (NULL != cd->opcbfunc) { + cd->opcbfunc(pmix20_convert_rc(status), cd->cbdata); + } + if (cd->active) { + cd->active = false; + } else { + OBJ_RELEASE(cd); + } +} + +static void _dereg_nspace(int sd, short args, void *cbdata) +{ + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + opal_pmix20_jobid_trkr_t *jptr; /* if we don't already have it, we can ignore this */ - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { - if (jptr->jobid == jobid) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { + if (jptr->jobid == cd->jobid) { /* found it - tell the server to deregister */ - PMIx_server_deregister_nspace(jptr->nspace); + cd->active = true; + PMIx_server_deregister_nspace(jptr->nspace, tdcbfunc, cd); + PMIX_WAIT_FOR_COMPLETION(cd->active); + OBJ_RELEASE(cd); /* now get rid of it from our list */ opal_list_remove_item(&mca_pmix_ext20_component.jobids, &jptr->super); OBJ_RELEASE(jptr); return; } } + /* must release the caller */ + tdcbfunc(PMIX_ERR_NOT_FOUND, cd); } -int pmix1_server_register_client(const opal_process_name_t *proc, +void pmix20_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix20_threadshift_t *cd; + + /* we must threadshift this request as it touches + * shared lists of objects */ + cd = OBJ_NEW(pmix20_threadshift_t); + cd->jobid = jobid; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + if (NULL == cbfunc) { + _dereg_nspace(0, 0, cd); + } else { + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _dereg_nspace, cd); + event_active(&cd->ev, EV_WRITE, 1); + } +} + +int pmix20_server_register_client(const opal_process_name_t *proc, uid_t uid, gid_t gid, void *server_object, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t rc; - pmix1_opcaddy_t *op; - - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->opcbfunc = cbfunc; - op->cbdata = cbdata; + pmix_proc_t p; + pmix20_opcaddy_t op; /* convert the jobid */ - (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, proc->jobid); - op->p.rank = proc->vpid; + (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid); + p.rank = proc->vpid; - rc = PMIx_server_register_client(&op->p, uid, gid, server_object, - opcbfunc, op); - if (PMIX_SUCCESS != rc) { - OBJ_RELEASE(op); + OBJ_CONSTRUCT(&op, pmix20_opcaddy_t); + op.active = true; + rc = PMIx_server_register_client(&p, uid, gid, server_object, + opcbfunc, (void*)&op); + if (PMIX_SUCCESS == rc) { + PMIX_WAIT_FOR_COMPLETION(op.active); + rc = op.status; } - return pmix1_convert_rc(rc); + OBJ_DESTRUCT(&op); + return pmix20_convert_rc(rc); } -void pmix1_server_deregister_client(const opal_process_name_t *proc) +static void _dereg_client(int sd, short args, void *cbdata) { - opal_pmix1_jobid_trkr_t *jptr; + pmix20_threadshift_t *cd = (pmix20_threadshift_t*)cbdata; + opal_pmix20_jobid_trkr_t *jptr; pmix_proc_t p; /* if we don't already have it, we can ignore this */ - OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix1_jobid_trkr_t) { - if (jptr->jobid == proc->jobid) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext20_component.jobids, opal_pmix20_jobid_trkr_t) { + if (jptr->jobid == cd->source->jobid) { /* found it - tell the server to deregister */ (void)strncpy(p.nspace, jptr->nspace, PMIX_MAX_NSLEN); - p.rank = proc->vpid; - PMIx_server_deregister_client(&p); - return; + p.rank = cd->source->vpid; + cd->active = true; + PMIx_server_deregister_client(&p, tdcbfunc, (void*)cd); + PMIX_WAIT_FOR_COMPLETION(cd->active); + break; } } + OBJ_RELEASE(cd); +} + +/* tell the local PMIx server to cleanup this client as it is + * done executing */ +void pmix20_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix20_threadshift_t *cd; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + cd = OBJ_NEW(pmix20_threadshift_t); + cd->source = proc; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + if (NULL == cbfunc) { + _dereg_client(0, 0, cd); + } else { + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _dereg_client, cd); + event_active(&cd->ev, EV_WRITE, 1); + } } - -int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env) +/* have the local PMIx server setup the environment for this client */ +int pmix20_server_setup_fork(const opal_process_name_t *proc, char ***env) { pmix_status_t rc; pmix_proc_t p; @@ -339,7 +413,7 @@ int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env) p.rank = proc->vpid; rc = PMIx_server_setup_fork(&p, env); - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } /* this is the call back up from the embedded PMIx server that @@ -348,23 +422,24 @@ int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env) static void dmdx_response(pmix_status_t status, char *data, size_t sz, void *cbdata) { int rc; - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix20_opcaddy_t *op = (pmix20_opcaddy_t*)cbdata; - rc = pmix1_convert_rc(status); + rc = pmix20_convert_rc(status); if (NULL != op->mdxcbfunc) { op->mdxcbfunc(rc, data, sz, op->cbdata, NULL, NULL); } OBJ_RELEASE(op); } -int pmix1_server_dmodex(const opal_process_name_t *proc, +/* request modex data for a local proc from the PMIx server */ +int pmix20_server_dmodex(const opal_process_name_t *proc, opal_pmix_modex_cbfunc_t cbfunc, void *cbdata) { - pmix1_opcaddy_t *op; + pmix20_opcaddy_t *op; pmix_status_t rc; /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix20_opcaddy_t); op->mdxcbfunc = cbfunc; op->cbdata = cbdata; @@ -377,50 +452,20 @@ int pmix1_server_dmodex(const opal_process_name_t *proc, if (PMIX_SUCCESS != rc) { OBJ_RELEASE(op); } - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } -int pmix1_server_notify_error(int status, - opal_list_t *procs, - opal_list_t *error_procs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +/* tell the PMIx server to notify its local clients of an event */ +int pmix20_server_notify_event(int status, + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { opal_value_t *kv; pmix_info_t *pinfo; - size_t sz, psz, esz, n; - pmix_proc_t *ps, *eps; + size_t sz, n; pmix_status_t rc; - pmix1_opcaddy_t *op; - opal_namelist_t *nm; - - /* convert the list of procs */ - if (NULL != procs) { - psz = opal_list_get_size(procs); - PMIX_PROC_CREATE(ps, psz); - n = 0; - OPAL_LIST_FOREACH(nm, procs, opal_namelist_t) { - (void)opal_snprintf_jobid(ps[n].nspace, PMIX_MAX_NSLEN, nm->name.jobid); - ps[n].rank = (int)nm->name.vpid; - ++n; - } - } else { - psz = 0; - ps = NULL; - } - if (NULL != error_procs) { - esz = opal_list_get_size(error_procs); - PMIX_PROC_CREATE(eps, esz); - n = 0; - OPAL_LIST_FOREACH(nm, error_procs, opal_namelist_t) { - (void)opal_snprintf_jobid(eps[n].nspace, PMIX_MAX_NSLEN, nm->name.jobid); - eps[n].rank = (int)nm->name.vpid; - ++n; - } - } else { - esz = 0; - eps = NULL; - } + pmix20_opcaddy_t *op; /* convert the list to an array of pmix_info_t */ if (NULL != info) { @@ -429,29 +474,35 @@ int pmix1_server_notify_error(int status, n = 0; OPAL_LIST_FOREACH(kv, info, opal_value_t) { (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, kv); + pmix20_value_load(&pinfo[n].value, kv); } } else { sz = 0; pinfo = NULL; } - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->procs = ps; - op->nprocs = psz; - op->error_procs = eps; - op->nerror_procs = esz; + op = OBJ_NEW(pmix20_opcaddy_t); op->info = pinfo; op->sz = sz; op->opcbfunc = cbfunc; op->cbdata = cbdata; + /* convert the jobid */ + if (NULL == source) { + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, OPAL_JOBID_INVALID); + op->p.rank = OPAL_VPID_INVALID; + } else { + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid); + op->p.rank = source->vpid; + } - rc = pmix1_convert_opalrc(status); - rc = PMIx_Notify_event(rc, ps, psz, eps, esz, + + rc = pmix20_convert_opalrc(status); + /* the range is irrelevant here as the server is passing + * the event down to its local clients */ + rc = PMIx_Notify_event(rc, &op->p, PMIX_RANGE_LOCAL, pinfo, sz, opcbfunc, op); if (PMIX_SUCCESS != rc) { OBJ_RELEASE(op); } - return pmix1_convert_rc(rc); + return pmix20_convert_rc(rc); } diff --git a/opal/mca/pmix/isolated/pmix_isolated.c b/opal/mca/pmix/isolated/pmix_isolated.c index 83adb79479..78522e4a44 100644 --- a/opal/mca/pmix/isolated/pmix_isolated.c +++ b/opal/mca/pmix/isolated/pmix_isolated.c @@ -106,8 +106,9 @@ const opal_pmix_base_module_t opal_pmix_isolated_module = { .resolve_peers = isolated_resolve_peers, .resolve_nodes = isolated_resolve_nodes, .get_version = isolated_get_version, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, + .register_evhandler = opal_pmix_base_register_handler, + .deregister_evhandler = opal_pmix_base_deregister_handler, + .notify_event = opal_pmix_base_notify_event, .store_local = isolated_store_local, .get_nspace = isolated_get_nspace, .register_jobid = isolated_register_jobid diff --git a/opal/mca/pmix/pmix.h b/opal/mca/pmix/pmix.h index b9cf9cbdcb..34108f8098 100644 --- a/opal/mca/pmix/pmix.h +++ b/opal/mca/pmix/pmix.h @@ -517,6 +517,9 @@ typedef int (*opal_pmix_base_module_resolve_nodes_fn_t)(opal_jobid_t jobid, char /************************************************************ * SERVER APIs * + * * + * These are calls that go down (or "south") from the ORTE * + * daemon into the PMIx server library * ************************************************************/ /* Initialize the server support library - must pass the callback @@ -595,7 +598,9 @@ typedef int (*opal_pmix_base_module_server_register_nspace_fn_t)(opal_jobid_t jo * can be freed. Note that the server will automatically * purge all info relating to any clients it has from * this nspace */ -typedef void (*opal_pmix_base_module_server_deregister_nspace_fn_t)(opal_jobid_t jobid); +typedef void (*opal_pmix_base_module_server_deregister_nspace_fn_t)(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); /* Register a client process with the PMIx server library. The * expected user ID and group ID of the child process helps the @@ -623,7 +628,9 @@ typedef int (*opal_pmix_base_module_server_register_client_fn_t)(const opal_proc * so there is no need to call this function during normal * finalize operations. Instead, this is provided for use * during exception operations */ -typedef void (*opal_pmix_base_module_server_deregister_client_fn_t)(const opal_process_name_t *proc); +typedef void (*opal_pmix_base_module_server_deregister_client_fn_t)(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); /* Setup the environment of a child process to be forked * by the host so it can correctly interact with the PMIx @@ -645,40 +652,34 @@ typedef int (*opal_pmix_base_module_server_dmodex_request_fn_t)(const opal_proce opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); -/* Report an error to a process for notification via any - * registered errhandler. The errhandler registration can be +/* Report an event to a process for notification via any + * registered event handler. The handler registration can be * called by both the server and the client application. On the - * server side, the errhandler is used to report errors detected + * server side, the handler is used to report events detected * by PMIx to the host server for handling. On the client side, - * the errhandler is used to notify the process of errors + * the handler is used to notify the process of events * reported by the server - e.g., the failure of another process. * * This function allows the host server to direct the server - * convenience library to notify all indicated local procs of - * an error. The error can be local, or anywhere in the cluster. - * The status indicates the error being reported. + * convenience library to notify all registered local procs of + * an event. The event can be local, or anywhere in the cluster. + * The status indicates the event being reported. * - * The first array of procs informs the server library as to which - * processes should be alerted - e.g., the processes that are in - * a directly-affected job or are connected to one that is affected. - * Passing a NULL for this array will indicate that all local procs - * are to be notified. - * - * The second array identifies the processes that will be impacted - * by the error. This could consist of a single process, or a number - * of processes. + * The source parameter informs the handler of the source that + * generated the event. This will be NULL if the event came + * from the external resource manager. * * The info array contains any further info the RM can and/or chooses * to provide. * * The callback function will be called upon completion of the - * notify_error function's actions. Note that any messages will + * notify_event function's actions. Note that any messages will * have been queued, but may not have been transmitted by this * time. Note that the caller is required to maintain the input - * data until the callback function has been executed! */ -typedef int (*opal_pmix_base_module_server_notify_error_fn_t)(int status, - opal_list_t *procs, - opal_list_t *error_procs, + * data until the callback function has been executed if this + * function returns OPAL_SUCCESS! */ +typedef int (*opal_pmix_base_module_server_notify_event_fn_t)(int status, + const opal_process_name_t *source, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -690,90 +691,76 @@ typedef int (*opal_pmix_base_module_server_notify_error_fn_t)(int status, /* get the version of the embedded library */ typedef const char* (*opal_pmix_base_module_get_version_fn_t)(void); -/* Register an errhandler to report errors. Three types of errors +/* Register an event handler to report event. Three types of events * can be reported: * * (a) those that occur within the client library, but are not * reportable via the API itself (e.g., loss of connection to - * the server). These errors typically occur during behind-the-scenes + * the server). These events typically occur during behind-the-scenes * non-blocking operations. * - * (b) job-related errors such as the failure of another process in + * (b) job-related events such as the failure of another process in * the job or in any connected job, impending failure of hardware * within the job's usage footprint, etc. * * (c) system notifications that are made available by the local * administrators * - * By default, only errors that directly affect the process and/or + * By default, only events that directly affect the process and/or * any process to which it is connected (via the PMIx_Connect call) * will be reported. Options to modify that behavior can be provided * in the info array * * Both the client application and the resource manager can register - * err handlers for specific errors. PMIx client/server calls the registered - * err handler upon receiving error notify notification (via PMIx_Notify_error) + * event handlers for specific events. PMIx client/server calls the registered + * event handler upon receiving event notify notification (via PMIx_Notify_event) * from the other end (Resource Manager/Client application). * - * Multiple err handlers can be registered for different errors. PMIX returns - * an integer reference to each register handler in the callback fn. The caller - * must retain the reference in order to deregister the errhandler. + * Multiple event handlers can be registered for different events. PMIX returns + * a size_t reference to each register handler in the callback fn. The caller + * must retain the reference in order to deregister the evhandler. * Modification of the notification behavior can be accomplished by - * deregistering the current errhandler, and then registering it + * deregistering the current evhandler, and then registering it * using a new set of info values. * + * A NULL for event_codes indicates registration as a default event handler + * * See pmix_types.h for a description of the notification function */ -typedef void (*opal_pmix_base_module_register_fn_t)(opal_list_t *info, - opal_pmix_notification_fn_t errhandler, - opal_pmix_errhandler_reg_cbfunc_t cbfunc, +typedef void (*opal_pmix_base_module_register_fn_t)(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, void *cbdata); -/* deregister the errhandler - * errhandler_ref is the reference returned by PMIx for the errhandler - * to pmix_errhandler_reg_cbfunc_t */ -typedef void (*opal_pmix_base_module_deregister_fn_t)(int errhandler, +/* deregister the evhandler + * evhandler_ref is the reference returned by PMIx for the evhandler + * to pmix_evhandler_reg_cbfunc_t */ +typedef void (*opal_pmix_base_module_deregister_fn_t)(size_t evhandler, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -/* Report an error to a process for notification via any - * registered errhandler. The errhandler registration can be - * called by both the server and the client application. On the - * server side, the errhandler is used to report errors detected - * by PMIx to the host server for handling. On the client side, - * the errhandler is used to notify the process of errors - * reported by the server - e.g., the failure of another process. - * - * This function allows the host server to direct the server - * convenience library to notify all indicated local procs of - * an error. The error can be local, or anywhere in the cluster. - * The status indicates the error being reported. +/* Report an event for notification via any + * registered evhandler. On the PMIx + * server side, this is used to report events detected + * by PMIx to the host server for handling and/or distribution. * * The client application can also call this function to notify the - * resource manager of an error it encountered. It can request the host - * server to notify the indicated processes about the error. + * resource manager of an event it detected. It can specify the + * range over which that notification should occur. * - * The first array of procs informs the server library as to which - * processes should be alerted - e.g., the processes that are in - * a directly-affected job or are connected to one that is affected. - * Passing a NULL for this array will indicate that all local procs - * are to be notified. - * - * The second array identifies the processes that will be impacted - * by the error. This could consist of a single process, or a number - * of processes. - * - * The info array contains any further info the RM can and/or chooses + * The info array contains any further info the caller can and/or chooses * to provide. * * The callback function will be called upon completion of the - * notify_error function's actions. Note that any messages will + * notify_event function's actions. Note that any messages will * have been queued, but may not have been transmitted by this * time. Note that the caller is required to maintain the input - * data until the callback function has been executed! + * data until the callback function has been executed if it + * returns OPAL_SUCCESS! */ -typedef int (*opal_pmix_base_module_notify_error_fn_t)(int status, - opal_list_t *procs, - opal_list_t *error_procs, +typedef int (*opal_pmix_base_module_notify_event_fn_t)(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -830,12 +817,12 @@ typedef struct { opal_pmix_base_module_server_deregister_client_fn_t server_deregister_client; opal_pmix_base_module_server_setup_fork_fn_t server_setup_fork; opal_pmix_base_module_server_dmodex_request_fn_t server_dmodex_request; - opal_pmix_base_module_server_notify_error_fn_t server_notify_error; + opal_pmix_base_module_server_notify_event_fn_t server_notify_event; /* Utility APIs */ opal_pmix_base_module_get_version_fn_t get_version; - opal_pmix_base_module_register_fn_t register_errhandler; - opal_pmix_base_module_deregister_fn_t deregister_errhandler; - opal_pmix_base_module_notify_error_fn_t notify_error; + opal_pmix_base_module_register_fn_t register_evhandler; + opal_pmix_base_module_deregister_fn_t deregister_evhandler; + opal_pmix_base_module_notify_event_fn_t notify_event; opal_pmix_base_module_store_fn_t store_local; opal_pmix_base_module_get_nspace_fn_t get_nspace; opal_pmix_base_module_register_jobid_fn_t register_jobid; diff --git a/opal/mca/pmix/pmix114/Makefile.am b/opal/mca/pmix/pmix114/Makefile.am deleted file mode 100644 index fdeaad4d08..0000000000 --- a/opal/mca/pmix/pmix114/Makefile.am +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2014-2016 Intel, Inc. All rights reserved. -# Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2015 Research Organization for Information Science -# and Technology (RIST). All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -EXTRA_DIST = autogen.subdirs - -SUBDIRS = pmix - -sources = \ - pmix1.h \ - pmix_pmix1_component.c \ - pmix_pmix1.c \ - pmix1_client.c \ - pmix1_server_south.c \ - pmix1_server_north.c - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_opal_pmix_pmix114_DSO -component_noinst = -component_install = mca_pmix_pmix114.la -else -component_noinst = libmca_pmix_pmix114.la -component_install = -endif - -mcacomponentdir = $(opallibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_pmix_pmix114_la_SOURCES = $(sources) -mca_pmix_pmix114_la_CFLAGS = $(opal_pmix_pmix114_CFLAGS) -mca_pmix_pmix114_la_CPPFLAGS = \ - -I$(srcdir)/pmix/include $(opal_pmix_pmix114_CPPFLAGS) -mca_pmix_pmix114_la_LDFLAGS = -module -avoid-version $(opal_pmix_pmix114_LDFLAGS) -mca_pmix_pmix114_la_LIBADD = $(opal_pmix_pmix114_LIBS) -mca_pmix_pmix114_la_DEPENDENCIES = $(mca_pmix_pmix114_la_LIBADD) - -noinst_LTLIBRARIES = $(component_noinst) -libmca_pmix_pmix114_la_SOURCES =$(sources) -libmca_pmix_pmix114_la_CFLAGS = $(opal_pmix_pmix114_CFLAGS) -libmca_pmix_pmix114_la_CPPFLAGS = -I$(srcdir)/pmix/include $(opal_pmix_pmix114_CPPFLAGS) -libmca_pmix_pmix114_la_LDFLAGS = -module -avoid-version $(opal_pmix_pmix114_LDFLAGS) -libmca_pmix_pmix114_la_LIBADD = $(opal_pmix_pmix114_LIBS) -libmca_pmix_pmix114_la_DEPENDENCIES = $(mca_pmix_pmix114_la_LIBADD) diff --git a/opal/mca/pmix/pmix114/configure.m4 b/opal/mca/pmix/pmix114/configure.m4 deleted file mode 100644 index 72eccd05ef..0000000000 --- a/opal/mca/pmix/pmix114/configure.m4 +++ /dev/null @@ -1,75 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2011-2013 Los Alamos National Security, LLC. -# All rights reserved. -# Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2013-2015 Intel, Inc. All rights reserved. -# Copyright (c) 2015 Research Organization for Information Science -# and Technology (RIST). All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_pmix_pmix114_CONFIG([action-if-found], [action-if-not-found]) -# ----------------------------------------------------------- -AC_DEFUN([MCA_opal_pmix_pmix114_CONFIG],[ - AC_CONFIG_FILES([opal/mca/pmix/pmix114/Makefile]) - - OPAL_VAR_SCOPE_PUSH([PMIX_VERSION opal_pmix_pmix114_save_CPPFLAGS opal_pmix_pmix114_save_LDFLAGS opal_pmix_pmix114_save_LIBS opal_pmix_pmix114_basedir opal_pmix_pmix114_save_cflags]) - - AS_IF([test "$opal_external_pmix_happy" = "yes"], - [AC_MSG_WARN([using an external pmix; disqualifiying this component]) - opal_pmix_pmix114_happy=0], - [PMIX_VERSION= - opal_pmix_pmix114_basedir=opal/mca/pmix/pmix114 - - opal_pmix_pmix114_save_CFLAGS=$CFLAGS - opal_pmix_pmix114_save_CPPFLAGS=$CPPFLAGS - opal_pmix_pmix114_save_LDFLAGS=$LDFLAGS - opal_pmix_pmix114_save_LIBS=$LIBS - - opal_pmix_pmix114_args="--enable-embedded-mode --with-pmix-symbol-prefix=opal_pmix_pmix114_ --disable-visibility --with-libevent-header=\\\"opal/mca/event/$opal_event_base_include\\\" --with-hwloc-header=\\\"$opal_hwloc_base_include\\\"" - AS_IF([test "$enable_debug" = "yes"], - [opal_pmix_pmix114_args="--enable-debug $opal_pmix_pmix114_args" - CFLAGS="$OPAL_CFLAGS_BEFORE_PICKY $OPAL_VISIBILITY_CFLAGS -g"], - [opal_pmix_pmix114_args="--disable-debug $opal_pmix_pmix114_args" - CFLAGS="$OPAL_CFLAGS_BEFORE_PICKY $OPAL_VISIBILITY_CFLAGS"]) - CPPFLAGS="-I$OPAL_TOP_SRCDIR -I$OPAL_TOP_BUILDDIR -I$OPAL_TOP_SRCDIR/opal/include -I$OPAL_TOP_BUILDDIR/opal/include $CPPFLAGS" - - OPAL_CONFIG_SUBDIR([$opal_pmix_pmix114_basedir/pmix], - [$opal_pmix_pmix114_args $opal_subdir_args 'CFLAGS=$CFLAGS' 'CPPFLAGS=$CPPFLAGS'], - [opal_pmix_pmix114_happy=1], [opal_pmix_pmix114_happy=0]) - - AS_IF([test $opal_pmix_pmix114_happy -eq 1], - [PMIX_VERSION="internal v`$srcdir/$opal_pmix_pmix114_basedir/pmix/config/pmix_get_version.sh $srcdir/$opal_pmix_pmix114_basedir/pmix/VERSION`" - # Build flags for our Makefile.am - opal_pmix_pmix114_LIBS='$(OPAL_TOP_BUILDDIR)/'"$opal_pmix_pmix114_basedir"'/pmix/libpmix.la' - opal_pmix_pmix114_CPPFLAGS='-I$(OPAL_TOP_BUILDDIR)/opal/mca/pmix/pmix114/pmix/include/pmix -I$(OPAL_TOP_BUILDDIR)/opal/mca/pmix/pmix114/pmix/include -I$(OPAL_TOP_BUILDDIR)/opal/mca/pmix/pmix114/pmix -I$(OPAL_TOP_SRCDIR)/opal/mca/pmix/pmix114/pmix' - AC_SUBST([opal_pmix_pmix114_LIBS]) - AC_SUBST([opal_pmix_pmix114_CPPFLAGS])]) - - CFLAGS=$opal_pmix_pmix114_save_CFLAGS - CPPFLAGS=$opal_pmix_pmix114_save_CPPFLAGS - LDFLAGS=$opal_pmix_pmix114_save_LDFLAGS - LIBS=$opal_pmix_pmix114_save_LIBS - ]) - - AS_IF([test $opal_pmix_pmix114_happy -eq 1], - [$1], - [$2]) - - OPAL_VAR_SCOPE_POP -])dnl diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/internal.h b/opal/mca/pmix/pmix114/pmix/src/buffer_ops/internal.h deleted file mode 100644 index 30179265e0..0000000000 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/internal.h +++ /dev/null @@ -1,474 +0,0 @@ -/* -*- C -*- - * - * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ -#ifndef PMIX_BFROP_INTERNAL_H_ -#define PMIX_BFROP_INTERNAL_H_ - -#include - - -#ifdef HAVE_SYS_TIME_H -#include /* for struct timeval */ -#endif - -#include "src/class/pmix_pointer_array.h" - -#include "buffer_ops.h" - -#ifdef HAVE_STRING_H -#include -#endif -#if PMIX_HAVE_HWLOC -#include PMIX_HWLOC_HEADER -#endif - -BEGIN_C_DECLS - -/* - * The default starting chunk size - */ -#define PMIX_BFROP_DEFAULT_INITIAL_SIZE 128 -/* - * The default threshold size when we switch from doubling the - * buffer size to addatively increasing it - */ -#define PMIX_BFROP_DEFAULT_THRESHOLD_SIZE 1024 - -/* - * Internal type corresponding to size_t. Do not use this in - * interface calls - use PMIX_SIZE instead. - */ -#if SIZEOF_SIZE_T == 1 -#define BFROP_TYPE_SIZE_T PMIX_UINT8 -#elif SIZEOF_SIZE_T == 2 -#define BFROP_TYPE_SIZE_T PMIX_UINT16 -#elif SIZEOF_SIZE_T == 4 -#define BFROP_TYPE_SIZE_T PMIX_UINT32 -#elif SIZEOF_SIZE_T == 8 -#define BFROP_TYPE_SIZE_T PMIX_UINT64 -#else -#error Unsupported size_t size! -#endif - -/* - * Internal type corresponding to bool. Do not use this in interface - * calls - use PMIX_BOOL instead. - */ -#if SIZEOF__BOOL == 1 -#define BFROP_TYPE_BOOL PMIX_UINT8 -#elif SIZEOF__BOOL == 2 -#define BFROP_TYPE_BOOL PMIX_UINT16 -#elif SIZEOF__BOOL == 4 -#define BFROP_TYPE_BOOL PMIX_UINT32 -#elif SIZEOF__BOOL == 8 -#define BFROP_TYPE_BOOL PMIX_UINT64 -#else -#error Unsupported bool size! -#endif - -/* - * Internal type corresponding to int and unsigned int. Do not use - * this in interface calls - use PMIX_INT / PMIX_UINT instead. - */ -#if SIZEOF_INT == 1 -#define BFROP_TYPE_INT PMIX_INT8 -#define BFROP_TYPE_UINT PMIX_UINT8 -#elif SIZEOF_INT == 2 -#define BFROP_TYPE_INT PMIX_INT16 -#define BFROP_TYPE_UINT PMIX_UINT16 -#elif SIZEOF_INT == 4 -#define BFROP_TYPE_INT PMIX_INT32 -#define BFROP_TYPE_UINT PMIX_UINT32 -#elif SIZEOF_INT == 8 -#define BFROP_TYPE_INT PMIX_INT64 -#define BFROP_TYPE_UINT PMIX_UINT64 -#else -#error Unsupported int size! -#endif - -/* - * Internal type corresponding to pid_t. Do not use this in interface - * calls - use PMIX_PID instead. - */ -#if SIZEOF_PID_T == 1 -#define BFROP_TYPE_PID_T PMIX_UINT8 -#elif SIZEOF_PID_T == 2 -#define BFROP_TYPE_PID_T PMIX_UINT16 -#elif SIZEOF_PID_T == 4 -#define BFROP_TYPE_PID_T PMIX_UINT32 -#elif SIZEOF_PID_T == 8 -#define BFROP_TYPE_PID_T PMIX_UINT64 -#else -#error Unsupported pid_t size! -#endif - -/* Unpack generic size macros */ -#define UNPACK_SIZE_MISMATCH(unpack_type, remote_type, ret) \ - do { \ - switch(remote_type) { \ - case PMIX_UINT8: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint8_t, remote_type); \ - break; \ - case PMIX_INT8: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int8_t, remote_type); \ - break; \ - case PMIX_UINT16: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint16_t, remote_type); \ - break; \ - case PMIX_INT16: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int16_t, remote_type); \ - break; \ - case PMIX_UINT32: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint32_t, remote_type); \ - break; \ - case PMIX_INT32: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int32_t, remote_type); \ - break; \ - case PMIX_UINT64: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint64_t, remote_type); \ - break; \ - case PMIX_INT64: \ - UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int64_t, remote_type); \ - break; \ - default: \ - ret = PMIX_ERR_NOT_FOUND; \ - } \ - } while (0) - -/* NOTE: do not need to deal with endianness here, as the unpacking of - the underling sender-side type will do that for us. Repeat: the - data in tmpbuf[] is already in host byte order. */ -#define UNPACK_SIZE_MISMATCH_FOUND(unpack_type, tmptype, tmpbfroptype) \ - do { \ - int32_t i; \ - tmptype *tmpbuf = (tmptype*)malloc(sizeof(tmptype) * (*num_vals)); \ - ret = pmix_bfrop_unpack_buffer(buffer, tmpbuf, num_vals, tmpbfroptype); \ - for (i = 0 ; i < *num_vals ; ++i) { \ - ((unpack_type*) dest)[i] = (unpack_type)(tmpbuf[i]); \ - } \ - free(tmpbuf); \ - } while (0) - - -/** - * Internal struct used for holding registered bfrop functions - */ -typedef struct { - pmix_object_t super; - /* type identifier */ - pmix_data_type_t odti_type; - /** Debugging string name */ - char *odti_name; - /** Pack function */ - pmix_bfrop_pack_fn_t odti_pack_fn; - /** Unpack function */ - pmix_bfrop_unpack_fn_t odti_unpack_fn; - /** copy function */ - pmix_bfrop_copy_fn_t odti_copy_fn; - /** print function */ - pmix_bfrop_print_fn_t odti_print_fn; -} pmix_bfrop_type_info_t; -PMIX_DECLSPEC PMIX_CLASS_DECLARATION(pmix_bfrop_type_info_t); - -/* - * globals needed within bfrop - */ -extern bool pmix_bfrop_initialized; -extern size_t pmix_bfrop_initial_size; -extern size_t pmix_bfrop_threshold_size; -extern pmix_pointer_array_t pmix_bfrop_types; -extern pmix_data_type_t pmix_bfrop_num_reg_types; - -/* macro for registering data types */ -#define PMIX_REGISTER_TYPE(n, t, p, u, c, pr) \ - do { \ - pmix_bfrop_type_info_t *_info; \ - _info = PMIX_NEW(pmix_bfrop_type_info_t); \ - _info->odti_name = strdup((n)); \ - _info->odti_type = (t); \ - _info->odti_pack_fn = (pmix_bfrop_pack_fn_t)(p); \ - _info->odti_unpack_fn = (pmix_bfrop_unpack_fn_t)(u); \ - _info->odti_copy_fn = (pmix_bfrop_copy_fn_t)(c) ; \ - _info->odti_print_fn = (pmix_bfrop_print_fn_t)(pr) ; \ - pmix_pointer_array_set_item(&pmix_bfrop_types, (t), _info); \ - ++pmix_bfrop_num_reg_types; \ - } while (0) - -/* - * Implementations of API functions - */ - - pmix_status_t pmix_bfrop_pack(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, - pmix_data_type_t type); - pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dest, - int32_t *max_num_vals, - pmix_data_type_t type); - - pmix_status_t pmix_bfrop_copy(void **dest, void *src, pmix_data_type_t type); - - pmix_status_t pmix_bfrop_print(char **output, char *prefix, void *src, pmix_data_type_t type); - - pmix_status_t pmix_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src); - -/* - * Specialized functions - */ -PMIX_DECLSPEC pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - -PMIX_DECLSPEC pmix_status_t pmix_bfrop_unpack_buffer(pmix_buffer_t *buffer, void *dst, - int32_t *num_vals, pmix_data_type_t type); - -/* - * Internal pack functions - */ - - pmix_status_t pmix_bfrop_pack_bool(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - pmix_status_t pmix_bfrop_pack_byte(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - pmix_status_t pmix_bfrop_pack_string(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_sizet(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_pid(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_pack_int(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_int16(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_int32(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_datatype(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_int64(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_pack_float(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_double(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_pack_timeval(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_time(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - -#if PMIX_HAVE_HWLOC -pmix_status_t pmix_bfrop_pack_topo(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -#endif -pmix_status_t pmix_bfrop_pack_value(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_proc(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_app(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_info(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_buf(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_kval(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_persist(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_bo(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_pack_pdata(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type); - -/* - * Internal unpack functions - */ -pmix_status_t pmix_bfrop_unpack_bool(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_byte(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_string(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_sizet(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_pid(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_unpack_int(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_int16(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_int32(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_datatype(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_int64(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_unpack_float(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_double(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_unpack_timeval(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_time(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); - -#if PMIX_HAVE_HWLOC -pmix_status_t pmix_bfrop_unpack_topo(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -#endif -pmix_status_t pmix_bfrop_unpack_value(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_proc(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_app(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_buf(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_kval(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_persist(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_bo(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); -pmix_status_t pmix_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type); - -/* - * Internal copy functions - */ - -pmix_status_t pmix_bfrop_std_copy(void **dest, void *src, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_copy_string(char **dest, char *src, pmix_data_type_t type); - -#if PMIX_HAVE_HWLOC -pmix_status_t pmix_bfrop_copy_topo(hwloc_topology_t *dest, - hwloc_topology_t src, - pmix_data_type_t type); -#endif -pmix_status_t pmix_bfrop_copy_value(pmix_value_t **dest, pmix_value_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_array(pmix_info_array_t **dest, pmix_info_array_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_buf(pmix_buffer_t **dest, pmix_buffer_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_kval(pmix_kval_t **dest, pmix_kval_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_bo(pmix_byte_object_t **dest, pmix_byte_object_t *src, - pmix_data_type_t type); -pmix_status_t pmix_bfrop_copy_pdata(pmix_pdata_t **dest, pmix_pdata_t *src, - pmix_data_type_t type); - -/* - * Internal print functions - */ -pmix_status_t pmix_bfrop_print_bool(char **output, char *prefix, bool *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_byte(char **output, char *prefix, uint8_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_string(char **output, char *prefix, char *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_size(char **output, char *prefix, size_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_pid(char **output, char *prefix, pid_t *src, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_print_int(char **output, char *prefix, int *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_int8(char **output, char *prefix, int8_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_int16(char **output, char *prefix, int16_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_int32(char **output, char *prefix, int32_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_int64(char **output, char *prefix, int64_t *src, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_print_uint(char **output, char *prefix, uint *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_uint8(char **output, char *prefix, uint8_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_uint16(char **output, char *prefix, uint16_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_uint32(char **output, char *prefix, uint32_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_uint64(char **output, char *prefix, uint64_t *src, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_print_float(char **output, char *prefix, float *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_double(char **output, char *prefix, double *src, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_print_timeval(char **output, char *prefix, struct timeval *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_time(char **output, char *prefix, time_t *src, pmix_data_type_t type); - -#if PMIX_HAVE_HWLOC -pmix_status_t pmix_bfrop_print_topo(char **output, char *prefix, - hwloc_topology_t src, pmix_data_type_t type); -#endif -pmix_status_t pmix_bfrop_print_value(char **output, char *prefix, pmix_value_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_array(char **output, char *prefix, - pmix_info_array_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_proc(char **output, char *prefix, - pmix_proc_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_app(char **output, char *prefix, - pmix_app_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_info(char **output, char *prefix, - pmix_info_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_buf(char **output, char *prefix, - pmix_buffer_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_kval(char **output, char *prefix, - pmix_kval_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_modex(char **output, char *prefix, - pmix_modex_data_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_persist(char **output, char *prefix, - pmix_persistence_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_bo(char **output, char *prefix, - pmix_byte_object_t *src, pmix_data_type_t type); -pmix_status_t pmix_bfrop_print_pdata(char **output, char *prefix, - pmix_pdata_t *src, pmix_data_type_t type); - -/* - * Internal helper functions - */ - -char* pmix_bfrop_buffer_extend(pmix_buffer_t *bptr, size_t bytes_to_add); - -bool pmix_bfrop_too_small(pmix_buffer_t *buffer, size_t bytes_reqd); - -pmix_bfrop_type_info_t* pmix_bfrop_find_type(pmix_data_type_t type); - -pmix_status_t pmix_bfrop_store_data_type(pmix_buffer_t *buffer, pmix_data_type_t type); - -pmix_status_t pmix_bfrop_get_data_type(pmix_buffer_t *buffer, pmix_data_type_t *type); - -END_C_DECLS - -#endif diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_ops.h b/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_ops.h deleted file mode 100644 index 689b04c04d..0000000000 --- a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_ops.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2015 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef PMIX_CLIENT_OPS_H -#define PMIX_CLIENT_OPS_H - -#include - - -#include "src/buffer_ops/buffer_ops.h" -#include "src/class/pmix_hash_table.h" -#include "src/usock/usock.h" - -BEGIN_C_DECLS - -typedef struct { - pmix_peer_t myserver; // messaging support to/from my server - pmix_list_t pending_requests; // list of pmix_cb_t pending data requests -} pmix_client_globals_t; - -extern pmix_client_globals_t pmix_client_globals; - -void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr); - -void pmix_client_register_errhandler(pmix_info_t info[], size_t ninfo, - pmix_notification_fn_t errhandler, - pmix_errhandler_reg_cbfunc_t cbfunc, - void *cbdata); - -void pmix_client_deregister_errhandler(int errhandler_ref, - pmix_op_cbfunc_t cbfunc, - void *cbdata); - -pmix_status_t pmix_client_notify_error(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_proc_t error_procs[], size_t error_nprocs, - pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); - -END_C_DECLS - -#endif /* PMIX_CLIENT_OPS_H */ diff --git a/opal/mca/pmix/pmix114/pmix/src/common/Makefile.am b/opal/mca/pmix/pmix114/pmix/src/common/Makefile.am deleted file mode 100644 index 099a99903e..0000000000 --- a/opal/mca/pmix/pmix114/pmix/src/common/Makefile.am +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2015 Intel, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -sources += \ - src/common/pmix_common.c diff --git a/opal/mca/pmix/pmix114/pmix/src/common/pmix_common.c b/opal/mca/pmix/pmix114/pmix/src/common/pmix_common.c deleted file mode 100644 index 2956de8273..0000000000 --- a/opal/mca/pmix/pmix114/pmix/src/common/pmix_common.c +++ /dev/null @@ -1,116 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2016 Mellanox Technologies, Inc. - * All rights reserved. - * Copyright (c) 2016 IBM Corporation, All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -#include - -#include -#include -#include - -#include -#include -#include - -#include "src/util/output.h" - -#include "src/client/pmix_client_ops.h" -#include "src/server/pmix_server_ops.h" -#include "src/include/pmix_globals.h" - -PMIX_EXPORT void PMIx_Register_errhandler(pmix_info_t info[], size_t ninfo, - pmix_notification_fn_t errhandler, - pmix_errhandler_reg_cbfunc_t cbfunc, - void *cbdata) -{ - if (pmix_globals.init_cntr <= 0) { - return; - } - - /* common err handler registration */ - if (pmix_globals.server) { - /* PMIX server: store the error handler, process info keys and call - * cbfunc with reference to the errhandler */ - pmix_output_verbose(2, pmix_globals.debug_output, - "registering server err handler"); - pmix_server_register_errhandler(info, ninfo, - errhandler, - cbfunc,cbdata); - - } else { - /* PMIX client: store the error handler, process info keys & - * call pmix_server_register_for_events, and call cbfunc with - * reference to the errhandler */ - pmix_output_verbose(2, pmix_globals.debug_output, - "registering client err handler"); - pmix_client_register_errhandler(info, ninfo, - errhandler, - cbfunc, cbdata); - } -} - -PMIX_EXPORT void PMIx_Deregister_errhandler(int errhandler_ref, - pmix_op_cbfunc_t cbfunc, - void *cbdata) -{ - if (pmix_globals.init_cntr <= 0) { - return; - } - - if (errhandler_ref < 0) { - return; - } - - /* common err handler registration */ - if (pmix_globals.server) { - /* PMIX server: store the error handler, process info keys and call - * cbfunc with reference to the errhandler */ - pmix_server_deregister_errhandler(errhandler_ref,cbfunc,cbdata); - pmix_output_verbose(2, pmix_globals.debug_output, - "deregistering server err handler"); - } else { - /* PMIX client: store the error handler, process info keys & - * call pmix_server_register_for_events, and call cbfunc with - * reference to the errhandler */ - pmix_client_deregister_errhandler(errhandler_ref, cbfunc, cbdata); - pmix_output_verbose(2, pmix_globals.debug_output, - "deregistering client err handler"); - } -} - -PMIX_EXPORT pmix_status_t PMIx_Notify_error(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_proc_t error_procs[], size_t error_nprocs, - pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) -{ - int rc; - - if (pmix_globals.init_cntr <= 0) { - return PMIX_ERR_INIT; - } - - if (pmix_globals.server) { - rc = pmix_server_notify_error(status, procs, nprocs, error_procs, - error_nprocs, info, ninfo, - cbfunc, cbdata); - pmix_output_verbose(0, pmix_globals.debug_output, - "pmix_server_notify_error error =%d, rc=%d", status, rc); - } else { - rc = pmix_client_notify_error(status, procs, nprocs, error_procs, - error_nprocs, info, ninfo, - cbfunc, cbdata); - pmix_output_verbose(0, pmix_globals.debug_output, - "pmix_client_notify_error error =%d, rc=%d", status, rc); - } - return rc; -} diff --git a/opal/mca/pmix/pmix114/pmix/src/util/error.c b/opal/mca/pmix/pmix114/pmix/src/util/error.c deleted file mode 100644 index 4da7542c34..0000000000 --- a/opal/mca/pmix/pmix114/pmix/src/util/error.c +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2012 Los Alamos National Security, LLC. - * All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include - - -#ifdef HAVE_STRING_H -#include -#endif -#include -#include -#ifdef HAVE_STDLIB_H -#include -#endif - -#include - -#include "src/util/error.h" -#include "src/include/pmix_globals.h" -#include "src/buffer_ops/buffer_ops.h" - -const char* PMIx_Error_string(pmix_status_t errnum) -{ - switch(errnum) { - case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - return "UNPACK-PAST-END"; - case PMIX_ERR_COMM_FAILURE: - return "COMM-FAILURE"; - case PMIX_ERR_NOT_IMPLEMENTED: - return "NOT-IMPLEMENTED"; - case PMIX_ERR_NOT_SUPPORTED: - return "NOT-SUPPORTED"; - case PMIX_ERR_NOT_FOUND: - return "NOT-FOUND"; - case PMIX_ERR_SERVER_NOT_AVAIL: - return "SERVER-NOT-AVAIL"; - case PMIX_ERR_INVALID_NAMESPACE: - return "INVALID-NAMESPACE"; - case PMIX_ERR_INVALID_SIZE: - return "INVALID-SIZE"; - case PMIX_ERR_INVALID_KEYVALP: - return "INVALID-KEYVAL"; - case PMIX_ERR_INVALID_NUM_PARSED: - return "INVALID-NUM-PARSED"; - - case PMIX_ERR_INVALID_ARGS: - return "INVALID-ARGS"; - case PMIX_ERR_INVALID_NUM_ARGS: - return "INVALID-NUM-ARGS"; - case PMIX_ERR_INVALID_LENGTH: - return "INVALID-LENGTH"; - case PMIX_ERR_INVALID_VAL_LENGTH: - return "INVALID-VAL-LENGTH"; - case PMIX_ERR_INVALID_VAL: - return "INVALID-VAL"; - case PMIX_ERR_INVALID_KEY_LENGTH: - return "INVALID-KEY-LENGTH"; - case PMIX_ERR_INVALID_KEY: - return "INVALID-KEY"; - case PMIX_ERR_INVALID_ARG: - return "INVALID-ARG"; - case PMIX_ERR_NOMEM: - return "NO-MEM"; - case PMIX_ERR_INIT: - return "INIT"; - - case PMIX_ERR_DATA_VALUE_NOT_FOUND: - return "DATA-VALUE-NOT-FOUND"; - case PMIX_ERR_OUT_OF_RESOURCE: - return "OUT-OF-RESOURCE"; - case PMIX_ERR_RESOURCE_BUSY: - return "RESOURCE-BUSY"; - case PMIX_ERR_BAD_PARAM: - return "BAD-PARAM"; - case PMIX_ERR_IN_ERRNO: - return "ERR-IN-ERRNO"; - case PMIX_ERR_UNREACH: - return "UNREACHABLE"; - case PMIX_ERR_TIMEOUT: - return "TIMEOUT"; - case PMIX_ERR_NO_PERMISSIONS: - return "NO-PERMISSIONS"; - case PMIX_ERR_PACK_MISMATCH: - return "PACK-MISMATCH"; - case PMIX_ERR_PACK_FAILURE: - return "PACK-FAILURE"; - - case PMIX_ERR_UNPACK_FAILURE: - return "UNPACK-FAILURE"; - case PMIX_ERR_UNPACK_INADEQUATE_SPACE: - return "UNPACK-INADEQUATE-SPACE"; - case PMIX_ERR_TYPE_MISMATCH: - return "TYPE-MISMATCH"; - case PMIX_ERR_PROC_ENTRY_NOT_FOUND: - return "PROC-ENTRY-NOT-FOUND"; - case PMIX_ERR_UNKNOWN_DATA_TYPE: - return "UNKNOWN-DATA-TYPE"; - case PMIX_ERR_WOULD_BLOCK: - return "WOULD-BLOCK"; - case PMIX_ERR_READY_FOR_HANDSHAKE: - return "READY-FOR-HANDSHAKE"; - case PMIX_ERR_HANDSHAKE_FAILED: - return "HANDSHAKE-FAILED"; - case PMIX_ERR_INVALID_CRED: - return "INVALID-CREDENTIAL"; - case PMIX_EXISTS: - return "EXISTS"; - case PMIX_ERR_SERVER_FAILED_REQUEST: - return "SERVER FAILED REQUEST"; - case PMIX_ERR_PROC_MIGRATE: - return "PROC-MIGRATE"; - case PMIX_ERR_PROC_CHECKPOINT: - return "PROC-CHECKPOINT-ERROR"; - case PMIX_ERR_PROC_RESTART: - return "PROC_RESTART"; - case PMIX_ERR_PROC_ABORTING: - return "PROC-ABORTING"; - case PMIX_ERR_PROC_REQUESTED_ABORT: - return "PROC-ABORT-REQUESTED"; - case PMIX_ERR_PROC_ABORTED: - return "PROC-ABORTED"; - case PMIX_ERR_DEBUGGER_RELEASE: - return "DEBUGGER-RELEASE"; - case PMIX_ERR_SILENT: - return "SILENT_ERROR"; - case PMIX_ERROR: - return "ERROR"; - case PMIX_SUCCESS: - return "SUCCESS"; - - } - return "ERROR STRING NOT FOUND"; -} - -void pmix_errhandler_invoke(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) -{ - /* We need to parse thru each registered handler and determine - * which one to call for the specific error */ - int i, idflt; - size_t j; - bool fired = false; - pmix_error_reg_info_t *errreg, *errdflt=NULL; - pmix_info_t *iptr; - - PMIX_INFO_CREATE(iptr, ninfo+1); - (void)strncpy(iptr[0].key, PMIX_ERROR_HANDLER_ID, PMIX_MAX_KEYLEN); - iptr[0].value.type = PMIX_INT; - if (NULL != info) { - for (j=0; j < ninfo; j++) { - PMIX_INFO_LOAD(&iptr[j+1], info[j].key, &info[j].value.data, info[j].value.type); - } - } - - for (i = 0; i < pmix_globals.errregs.size; i++) { - if (NULL == (errreg = (pmix_error_reg_info_t*) pmix_pointer_array_get_item(&pmix_globals.errregs, i))) { - continue; - } - if (NULL == errreg->info || 0 == errreg->ninfo) { - // this is a general err handler - we will call it if there is no better match - errdflt = errreg; - idflt = i; - continue; - } - iptr[0].value.data.integer = i; - /* match error name key first */ - for (j = 0; j < errreg->ninfo; j++) { - if ((0 == strcmp(errreg->info[j].key, PMIX_ERROR_NAME)) && - (status == errreg->info[j].value.data.int32)) { - iptr[0].value.data.integer = i; - errreg->errhandler(status, procs, nprocs, iptr, ninfo+1); - fired = true; - break; - } - } - } - - /* if nothing fired and we found a general err handler, then fire it */ - if (!fired && NULL != errdflt) { - iptr[0].value.data.integer = idflt; - errdflt->errhandler(status, procs, nprocs, iptr, ninfo+1); - } - /* cleanup */ - PMIX_INFO_FREE(iptr, ninfo+1); -} - -pmix_status_t pmix_lookup_errhandler(pmix_notification_fn_t err, - int *index) -{ - int i; - pmix_error_reg_info_t *errreg; - pmix_status_t rc = PMIX_ERR_NOT_FOUND; - - for (i = 0; i < pmix_pointer_array_get_size(&pmix_globals.errregs) ; i++) { - errreg = (pmix_error_reg_info_t*)pmix_pointer_array_get_item(&pmix_globals.errregs, i); - if ((NULL != errreg) && (err == errreg->errhandler)) { - *index = i; - rc = PMIX_SUCCESS; - break; - } - } - return rc; -} - -pmix_status_t pmix_add_errhandler(pmix_notification_fn_t err, - pmix_info_t *info, int ninfo, - int *index) -{ - int i; - pmix_status_t rc = PMIX_SUCCESS; - pmix_error_reg_info_t *errreg; - - errreg = PMIX_NEW(pmix_error_reg_info_t); - errreg->errhandler = err; - errreg->ninfo = ninfo; - if (NULL != info && 0 < ninfo) { - PMIX_INFO_CREATE(errreg->info, ninfo); - for (i=0; i < ninfo; i++) { - (void)strncpy(errreg->info[i].key, info[i].key, PMIX_MAX_KEYLEN); - pmix_value_xfer(&errreg->info[i].value, &info[i].value); - } - } - *index = pmix_pointer_array_add(&pmix_globals.errregs, errreg); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_add_errhandler index =%d", *index); - if (*index < 0) { - PMIX_RELEASE(errreg); - rc = PMIX_ERROR; - } - return rc; -} - -pmix_status_t pmix_remove_errhandler(int errhandler_ref) -{ - int rc = PMIX_SUCCESS; - pmix_error_reg_info_t *errreg; - - errreg = (pmix_error_reg_info_t*)pmix_pointer_array_get_item(&pmix_globals.errregs, - errhandler_ref); - if (NULL != errreg) { - PMIX_RELEASE(errreg); - pmix_pointer_array_set_item(&pmix_globals.errregs, errhandler_ref, NULL); - } else { - rc = PMIX_ERR_NOT_FOUND; - } - return rc; -} - -void pmix_get_errorgroup(pmix_status_t status, char *pmix_error_group) -{ - switch(status) { - case PMIX_ERR_UNREACH: - case PMIX_ERR_COMM_FAILURE: - case PMIX_ERR_SERVER_NOT_AVAIL: - case PMIX_ERR_TIMEOUT: - case PMIX_ERR_PACK_FAILURE: - case PMIX_ERR_UNPACK_FAILURE: - (void)strncpy(pmix_error_group, PMIX_ERROR_GROUP_COMM, PMIX_MAX_KEYLEN); - break; - case PMIX_ERR_OUT_OF_RESOURCE: - case PMIX_ERR_RESOURCE_BUSY: - case PMIX_ERR_NOMEM: - (void)strncpy(pmix_error_group, PMIX_ERROR_GROUP_RESOURCE, PMIX_MAX_KEYLEN); - break; - case PMIX_ERR_PROC_MIGRATE: - case PMIX_ERR_PROC_CHECKPOINT: - case PMIX_ERR_PROC_RESTART: - (void)strncpy(pmix_error_group, PMIX_ERROR_GROUP_MIGRATE, PMIX_MAX_KEYLEN); - break; - case PMIX_ERR_PROC_ABORTING: - case PMIX_ERR_PROC_REQUESTED_ABORT: - case PMIX_ERR_PROC_ABORTED: - (void)strncpy(pmix_error_group, PMIX_ERROR_GROUP_ABORT, PMIX_MAX_KEYLEN); - break; - default: - (void)strncpy(pmix_error_group, PMIX_ERROR_GROUP_GENERAL, PMIX_MAX_KEYLEN); - } -} diff --git a/opal/mca/pmix/pmix114/pmix/src/util/error.h b/opal/mca/pmix/pmix114/pmix/src/util/error.h deleted file mode 100644 index e9a99c7538..0000000000 --- a/opal/mca/pmix/pmix114/pmix/src/util/error.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef PMIX_UTIL_ERROR_H -#define PMIX_UTIL_ERROR_H - -#include - - -#include -#include "src/util/output.h" - -BEGIN_C_DECLS - -#define PMIX_ERROR_LOG(r) \ - do { \ - if (PMIX_ERR_SILENT != (r)) { \ - pmix_output(0, "PMIX ERROR: %s in file %s at line %d", \ - PMIx_Error_string((r)), __FILE__, __LINE__); \ - } \ - } while (0) - -#define PMIX_REPORT_ERROR(e) \ - pmix_errhandler_invoke(e, NULL, 0, NULL, 0) - -void pmix_errhandler_invoke(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo); - -pmix_status_t pmix_lookup_errhandler(pmix_notification_fn_t err, - int *index); - -pmix_status_t pmix_add_errhandler(pmix_notification_fn_t err, - pmix_info_t *info, int ninfo, - int *index); - -pmix_status_t pmix_remove_errhandler(int errhandler_ref); - -void pmix_get_errorgroup(pmix_status_t status, char *pmix_error_group); - -PMIX_DECLSPEC pmix_status_t pmix_lookup_errhandler(pmix_notification_fn_t err, - int *index); - -PMIX_DECLSPEC pmix_status_t pmix_add_errhandler(pmix_notification_fn_t err, - pmix_info_t *info, int ninfo, - int *index); - -PMIX_DECLSPEC pmix_status_t pmix_remove_errhandler(int errhandler_ref); - -PMIX_DECLSPEC void pmix_get_errorgroup ( pmix_status_t status, char *pmix_error_group); - -END_C_DECLS - -#endif /* PMIX_UTIL_ERROR_H */ diff --git a/opal/mca/pmix/pmix114/pmix/test/test_error.c b/opal/mca/pmix/pmix114/pmix/test/test_error.c deleted file mode 100644 index f56da9ce66..0000000000 --- a/opal/mca/pmix/pmix114/pmix/test/test_error.c +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2015 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ -#include -#include "test_error.h" -#include "test_common.h" - -#define MAX_ERR_HANDLERS 5 -#define TEST_NOTIFY PMIX_ERR_TIMEOUT -static bool done; -static void comfail_errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) -{ - TEST_ERROR(("comfail errhandler called for error status = %d nprocs =%d ninfo = %d", - status, nprocs, ninfo)); -} - -static void timeout_errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) -{ - TEST_ERROR(("timeout errhandler called for error status = %d nprocs = %d ninfo = %d", - status, nprocs, ninfo)); -} - -static void op1_callbk(pmix_status_t status, - void *cbdata) -{ - TEST_VERBOSE(( "op1_callbk CALLED WITH STATUS %d", status)); - done = true; -} - -static void errhandler_reg_callbk1 (pmix_status_t status, - int errhandler_ref, - void *cbdata) -{ - int *ref = (int*) cbdata; - *ref = errhandler_ref; - TEST_VERBOSE(("PMIX client ERRHANDLER REGISTRATION CALLED WITH STATUS %d, ref=%d", - status, *ref, errhandler_ref)); - -} - -int test_error(char *my_nspace, int my_rank, test_params params) -{ - pmix_info_t *info; - size_t ninfo; - int errhandler_refs[MAX_ERR_HANDLERS]; - int value; - struct timespec ts; - TEST_VERBOSE(("test-error: running error handling test cases")); - /* register specific client error handlers and test their invocation - * by trigerring events from server side*/ - ninfo = 1; - value = PMIX_ERR_TIMEOUT; - PMIX_INFO_CREATE(info, ninfo); - (void)strncpy(info[0].key, PMIX_ERROR_NAME, PMIX_MAX_KEYLEN); - pmix_value_load(&info[0].value, &value, PMIX_INT); - PMIx_Register_errhandler(info, 1, timeout_errhandler, errhandler_reg_callbk1, &errhandler_refs[0]); - /* reg a handler for comm errors */ - (void)strncpy(info[0].key, PMIX_ERROR_GROUP_COMM, PMIX_MAX_KEYLEN); - value = 1; - pmix_value_load(&info[0].value, &value, PMIX_BOOL); - PMIx_Register_errhandler(info, 1, comfail_errhandler, errhandler_reg_callbk1, &errhandler_refs[1]); - /* inject error from client */ - done = false; - /* change error value to test other error notifications */ - PMIx_Notify_error(TEST_NOTIFY, - NULL, 0, - NULL, 0, NULL, 0, - op1_callbk, NULL); - while(!done) { - ts.tv_sec = 0; - ts.tv_nsec = 100000; - nanosleep(&ts, NULL); - } - done = false; - /* dereg all handlers*/ - PMIx_Deregister_errhandler( errhandler_refs[0], op1_callbk, NULL); - /* loop until we get callback */ - while(!done) { - ts.tv_sec = 0; - ts.tv_nsec = 100000; - nanosleep(&ts, NULL); - } - done = false; - PMIx_Deregister_errhandler( errhandler_refs[1], op1_callbk, NULL); - /* loop until we get callback */ - while(!done) { - ts.tv_sec = 0; - ts.tv_nsec = 100000; - nanosleep(&ts, NULL); - } - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix114/pmix1.h b/opal/mca/pmix/pmix114/pmix1.h deleted file mode 100644 index 4a126e359f..0000000000 --- a/opal/mca/pmix/pmix114/pmix1.h +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_PMIX_PMIX1_H -#define MCA_PMIX_PMIX1_H - -#include "opal_config.h" - -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_SYS_UN_H -#include -#endif - -#include "opal/mca/mca.h" -#include "opal/mca/event/event.h" -#include "opal/util/proc.h" - -#include "opal/mca/pmix/pmix.h" -#include "opal/mca/pmix/pmix_server.h" -#include "opal/mca/pmix/pmix114/pmix/include/pmix_server.h" -#include "opal/mca/pmix/pmix114/pmix/include/pmix/pmix_common.h" - -BEGIN_C_DECLS - -typedef struct { - opal_pmix_base_component_t super; - opal_list_t jobids; - bool native_launch; -} mca_pmix_pmix1_component_t; - -OPAL_DECLSPEC extern mca_pmix_pmix1_component_t mca_pmix_pmix114_component; - -OPAL_DECLSPEC extern const opal_pmix_base_module_t opal_pmix_pmix114_module; - -/**** INTERNAL OBJECTS ****/ -typedef struct { - opal_list_item_t super; - opal_jobid_t jobid; - char nspace[PMIX_MAX_NSLEN + 1]; -} opal_pmix1_jobid_trkr_t; -OBJ_CLASS_DECLARATION(opal_pmix1_jobid_trkr_t); - -typedef struct { - opal_object_t super; - pmix_proc_t p; - pmix_proc_t *procs; - size_t nprocs; - pmix_proc_t *error_procs; - size_t nerror_procs; - pmix_info_t *info; - size_t ninfo; - pmix_app_t *apps; - size_t sz; - opal_pmix_op_cbfunc_t opcbfunc; - opal_pmix_modex_cbfunc_t mdxcbfunc; - opal_pmix_value_cbfunc_t valcbfunc; - opal_pmix_lookup_cbfunc_t lkcbfunc; - opal_pmix_spawn_cbfunc_t spcbfunc; - void *cbdata; -} pmix1_opcaddy_t; -OBJ_CLASS_DECLARATION(pmix1_opcaddy_t); - -typedef struct { - opal_object_t super; - opal_list_t procs; - opal_list_t info; - opal_list_t apps; - pmix_op_cbfunc_t opcbfunc; - pmix_dmodex_response_fn_t dmdxfunc; - pmix_modex_cbfunc_t mdxcbfunc; - pmix_lookup_cbfunc_t lkupcbfunc; - pmix_spawn_cbfunc_t spwncbfunc; - void *cbdata; - opal_pmix_release_cbfunc_t odmdxfunc; - void *ocbdata; -} pmix1_opalcaddy_t; -OBJ_CLASS_DECLARATION(pmix1_opalcaddy_t); - - -/**** CLIENT FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC int pmix1_client_init(void); -OPAL_MODULE_DECLSPEC int pmix1_client_finalize(void); -OPAL_MODULE_DECLSPEC int pmix1_initialized(void); -OPAL_MODULE_DECLSPEC int pmix1_abort(int flag, const char *msg, - opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_commit(void); -OPAL_MODULE_DECLSPEC int pmix1_fence(opal_list_t *procs, int collect_data); -OPAL_MODULE_DECLSPEC int pmix1_fencenb(opal_list_t *procs, int collect_data, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_put(opal_pmix_scope_t scope, - opal_value_t *val); -OPAL_MODULE_DECLSPEC int pmix1_get(const opal_process_name_t *proc, const char *key, - opal_list_t *info, opal_value_t **val); -OPAL_MODULE_DECLSPEC int pmix1_getnb(const opal_process_name_t *proc, const char *key, - opal_list_t *info, - opal_pmix_value_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_publish(opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_publishnb(opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_lookup(opal_list_t *data, opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_lookupnb(char **keys, opal_list_t *info, - opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_unpublish(char **keys, opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_unpublishnb(char **keys, opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid); -OPAL_MODULE_DECLSPEC int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, - opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_connect(opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_connectnb(opal_list_t *procs, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_disconnect(opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_disconnectnb(opal_list_t *procs, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, - opal_list_t *procs); -OPAL_MODULE_DECLSPEC int pmix1_resolve_nodes(opal_jobid_t jobid, char **nodelist); - -/**** COMMON FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC int pmix1_store_local(const opal_process_name_t *proc, - opal_value_t *val); - -/**** SERVER SOUTHBOUND FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC int pmix1_server_init(opal_pmix_server_module_t *module, - opal_list_t *info); -OPAL_MODULE_DECLSPEC int pmix1_server_finalize(void); -OPAL_MODULE_DECLSPEC int pmix1_server_gen_regex(const char *input, char **regex); -OPAL_MODULE_DECLSPEC int pmix1_server_gen_ppn(const char *input, char **ppn); -OPAL_MODULE_DECLSPEC int pmix1_server_register_nspace(opal_jobid_t jobid, - int nlocalprocs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata); -OPAL_MODULE_DECLSPEC void pmix1_server_deregister_nspace(opal_jobid_t jobid); -OPAL_MODULE_DECLSPEC int pmix1_server_register_client(const opal_process_name_t *proc, - uid_t uid, gid_t gid, - void *server_object, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata); -OPAL_MODULE_DECLSPEC void pmix1_server_deregister_client(const opal_process_name_t *proc); -OPAL_MODULE_DECLSPEC int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env); -OPAL_MODULE_DECLSPEC int pmix1_server_dmodex(const opal_process_name_t *proc, - opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); -OPAL_MODULE_DECLSPEC int pmix1_server_notify_error(int status, - opal_list_t *procs, - opal_list_t *error_procs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata); - - -/**** COMPONENT UTILITY FUNCTIONS ****/ -OPAL_MODULE_DECLSPEC pmix_status_t pmix1_convert_opalrc(int rc); -OPAL_MODULE_DECLSPEC int pmix1_convert_rc(pmix_status_t rc); -OPAL_MODULE_DECLSPEC pmix_scope_t pmix1_convert_opalscope(opal_pmix_scope_t scope); -OPAL_MODULE_DECLSPEC void pmix1_value_load(pmix_value_t *v, - opal_value_t *kv); -OPAL_MODULE_DECLSPEC int pmix1_value_unload(opal_value_t *kv, - const pmix_value_t *v); - -END_C_DECLS - -#endif /* MCA_PMIX_PMIX1_H */ diff --git a/opal/mca/pmix/pmix114/pmix1_server_south.c b/opal/mca/pmix/pmix114/pmix1_server_south.c deleted file mode 100644 index 8ed71d45de..0000000000 --- a/opal/mca/pmix/pmix114/pmix1_server_south.c +++ /dev/null @@ -1,440 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2014 Mellanox Technologies, Inc. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" -#include "opal/constants.h" -#include "opal/types.h" - -#ifdef HAVE_STRING_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif - -#include "opal/dss/dss.h" -#include "opal/mca/event/event.h" -#include "opal/mca/hwloc/base/base.h" -#include "opal/runtime/opal.h" -#include "opal/runtime/opal_progress_threads.h" -#include "opal/util/argv.h" -#include "opal/util/error.h" -#include "opal/util/output.h" -#include "opal/util/proc.h" -#include "opal/util/show_help.h" -#include "opal/mca/pmix/base/base.h" -#include "pmix1.h" - -#include "pmix.h" -#include "pmix_server.h" - -/**** S.O.U.T.H.B.O.U.N.D I.N.T.E.R.F.A.C.E.S ****/ - -/* These are the interfaces used by the OMPI/ORTE/OPAL layer to call - * down into the embedded PMIx server. */ - -extern pmix_server_module_t pmix114_module; -extern opal_pmix_server_module_t *pmix114_host_module; -static char *dbgvalue=NULL; -static int errhdler_ref = 0; - -static void release_cbfunc(void *cbdata) -{ - pmix1_opalcaddy_t *cd = (pmix1_opalcaddy_t*)cbdata; - OBJ_RELEASE(cd); -} -static void myerr(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) -{ - int rc; - opal_namelist_t *nm; - opal_value_t *iptr; - size_t n; - pmix1_opalcaddy_t *cd; - - /* convert the incoming status */ - rc = pmix1_convert_rc(status); - - /* setup the caddy */ - cd = OBJ_NEW(pmix1_opalcaddy_t); - - /* convert the array of procs */ - for (n=0; n < nprocs; n++) { - nm = OBJ_NEW(opal_namelist_t); - nm->name.jobid = strtoul(procs[n].nspace, NULL, 10); - nm->name.vpid = procs[n].rank; - opal_list_append(&cd->procs, &nm->super); - } - - /* convert the array of info */ - for (n=0; n < ninfo; n++) { - iptr = OBJ_NEW(opal_value_t); - iptr->key = strdup(info[n].key); - pmix1_value_unload(iptr, &info[n].value); - opal_list_append(&cd->info, &iptr->super); - } - - /* call the base errhandler */ - opal_pmix_base_errhandler(rc, &cd->procs, &cd->info, release_cbfunc, cd); -} - -static void errreg_cbfunc(pmix_status_t status, - int errhandler_ref, - void *cbdata) -{ - errhdler_ref = errhandler_ref; - opal_output_verbose(5, opal_pmix_base_framework.framework_output, - "PMIX server errreg_cbfunc - error handler registered status=%d, reference=%d", - status, errhandler_ref); -} - -int pmix1_server_init(opal_pmix_server_module_t *module, - opal_list_t *info) -{ - pmix_status_t rc; - int dbg; - opal_value_t *kv; - pmix_info_t *pinfo; - size_t sz, n; - - if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) { - asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg); - putenv(dbgvalue); - } - - /* convert the list to an array of pmix_info_t */ - if (NULL != info) { - sz = opal_list_get_size(info); - PMIX_INFO_CREATE(pinfo, sz); - n = 0; - OPAL_LIST_FOREACH(kv, info, opal_value_t) { - (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, kv); - ++n; - } - } else { - sz = 0; - pinfo = NULL; - } - - if (PMIX_SUCCESS != (rc = PMIx_server_init(&pmix114_module, pinfo, sz))) { - PMIX_INFO_FREE(pinfo, sz); - return pmix1_convert_rc(rc); - } - PMIX_INFO_FREE(pinfo, sz); - - /* record the host module */ - pmix114_host_module = module; - - /* register the errhandler */ - PMIx_Register_errhandler(NULL, 0, myerr, errreg_cbfunc, NULL); - return OPAL_SUCCESS; -} - -int pmix1_server_finalize(void) -{ - pmix_status_t rc; - - /* deregister the errhandler */ - PMIx_Deregister_errhandler(errhdler_ref, NULL, NULL); - - rc = PMIx_server_finalize(); - return pmix1_convert_rc(rc); -} - -int pmix1_server_gen_regex(const char *input, char **regex) -{ - pmix_status_t rc; - - rc = PMIx_generate_regex(input, regex); - return pmix1_convert_rc(rc); -} - - -int pmix1_server_gen_ppn(const char *input, char **ppn) -{ - pmix_status_t rc; - - rc = PMIx_generate_ppn(input, ppn); - return pmix1_convert_rc(rc); -} - -static void opcbfunc(pmix_status_t status, void *cbdata) -{ - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; - - if (NULL != op->opcbfunc) { - op->opcbfunc(pmix1_convert_rc(status), op->cbdata); - } - OBJ_RELEASE(op); -} - -int pmix1_server_register_nspace(opal_jobid_t jobid, - int nlocalprocs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata) -{ - opal_value_t *kv, *k2; - pmix_info_t *pinfo, *pmap; - size_t sz, szmap, m, n; - char nspace[PMIX_MAX_NSLEN]; - pmix_status_t rc; - pmix1_opcaddy_t *op; - opal_list_t *pmapinfo; - opal_pmix1_jobid_trkr_t *job; - - /* convert the jobid */ - (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid); - - /* store this job in our list of known nspaces */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); - (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); - job->jobid = jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); - - /* convert the list to an array of pmix_info_t */ - if (NULL != info) { - sz = opal_list_get_size(info); - PMIX_INFO_CREATE(pinfo, sz); - n = 0; - OPAL_LIST_FOREACH(kv, info, opal_value_t) { - (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); - if (0 == strcmp(kv->key, OPAL_PMIX_PROC_DATA)) { - pinfo[n].value.type = PMIX_INFO_ARRAY; - /* the value contains a list of values - convert - * that list to another array */ - pmapinfo = (opal_list_t*)kv->data.ptr; - szmap = opal_list_get_size(pmapinfo); - PMIX_INFO_CREATE(pmap, szmap); - pinfo[n].value.data.array.array = (struct pmix_info_t *)pmap; - pinfo[n].value.data.array.size = szmap; - m = 0; - OPAL_LIST_FOREACH(k2, pmapinfo, opal_value_t) { - (void)strncpy(pmap[m].key, k2->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pmap[m].value, k2); - ++m; - } - } else { - pmix1_value_load(&pinfo[n].value, kv); - } - ++n; - } - } else { - sz = 0; - pinfo = NULL; - } - - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->info = pinfo; - op->sz = sz; - op->opcbfunc = cbfunc; - op->cbdata = cbdata; - rc = PMIx_server_register_nspace(nspace, nlocalprocs, pinfo, sz, - opcbfunc, op); - if (PMIX_SUCCESS != rc) { - OBJ_RELEASE(op); - } - return pmix1_convert_rc(rc); -} - -void pmix1_server_deregister_nspace(opal_jobid_t jobid) -{ - opal_pmix1_jobid_trkr_t *jptr; - - /* if we don't already have it, we can ignore this */ - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { - if (jptr->jobid == jobid) { - /* found it - tell the server to deregister */ - PMIx_server_deregister_nspace(jptr->nspace); - /* now get rid of it from our list */ - opal_list_remove_item(&mca_pmix_pmix114_component.jobids, &jptr->super); - OBJ_RELEASE(jptr); - return; - } - } -} - -int pmix1_server_register_client(const opal_process_name_t *proc, - uid_t uid, gid_t gid, - void *server_object, - opal_pmix_op_cbfunc_t cbfunc, - void *cbdata) -{ - pmix_status_t rc; - pmix1_opcaddy_t *op; - - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->opcbfunc = cbfunc; - op->cbdata = cbdata; - - /* convert the jobid */ - (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, proc->jobid); - op->p.rank = proc->vpid; - - rc = PMIx_server_register_client(&op->p, uid, gid, server_object, - opcbfunc, op); - if (PMIX_SUCCESS != rc) { - OBJ_RELEASE(op); - } - return pmix1_convert_rc(rc); -} - -void pmix1_server_deregister_client(const opal_process_name_t *proc) -{ - opal_pmix1_jobid_trkr_t *jptr; - pmix_proc_t p; - - /* if we don't already have it, we can ignore this */ - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { - if (jptr->jobid == proc->jobid) { - /* found it - tell the server to deregister */ - (void)strncpy(p.nspace, jptr->nspace, PMIX_MAX_NSLEN); - p.rank = proc->vpid; - PMIx_server_deregister_client(&p); - return; - } - } -} - - -int pmix1_server_setup_fork(const opal_process_name_t *proc, char ***env) -{ - pmix_status_t rc; - pmix_proc_t p; - - /* convert the jobid */ - (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid); - p.rank = proc->vpid; - - rc = PMIx_server_setup_fork(&p, env); - return pmix1_convert_rc(rc); -} - -/* this is the call back up from the embedded PMIx server that - * will contain the returned data. Note that the embedded server - * "owns" the data and will free it upon return from this function */ -static void dmdx_response(pmix_status_t status, char *data, size_t sz, void *cbdata) -{ - int rc; - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; - - rc = pmix1_convert_rc(status); - if (NULL != op->mdxcbfunc) { - op->mdxcbfunc(rc, data, sz, op->cbdata, NULL, NULL); - } - OBJ_RELEASE(op); -} - -int pmix1_server_dmodex(const opal_process_name_t *proc, - opal_pmix_modex_cbfunc_t cbfunc, void *cbdata) -{ - pmix1_opcaddy_t *op; - pmix_status_t rc; - - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->mdxcbfunc = cbfunc; - op->cbdata = cbdata; - - /* convert the jobid */ - (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, proc->jobid); - op->p.rank = proc->vpid; - - /* find the internally-cached data for this proc */ - rc = PMIx_server_dmodex_request(&op->p, dmdx_response, op); - if (PMIX_SUCCESS != rc) { - OBJ_RELEASE(op); - } - return pmix1_convert_rc(rc); -} - -int pmix1_server_notify_error(int status, - opal_list_t *procs, - opal_list_t *error_procs, - opal_list_t *info, - opal_pmix_op_cbfunc_t cbfunc, void *cbdata) -{ - opal_value_t *kv; - pmix_info_t *pinfo; - size_t sz, psz, esz, n; - pmix_proc_t *ps, *eps; - pmix_status_t rc; - pmix1_opcaddy_t *op; - opal_namelist_t *nm; - - /* convert the list of procs */ - if (NULL != procs) { - psz = opal_list_get_size(procs); - PMIX_PROC_CREATE(ps, psz); - n = 0; - OPAL_LIST_FOREACH(nm, procs, opal_namelist_t) { - (void)opal_snprintf_jobid(ps[n].nspace, PMIX_MAX_NSLEN, nm->name.jobid); - ps[n].rank = (int)nm->name.vpid; - ++n; - } - } else { - psz = 0; - ps = NULL; - } - if (NULL != error_procs) { - esz = opal_list_get_size(error_procs); - PMIX_PROC_CREATE(eps, esz); - n = 0; - OPAL_LIST_FOREACH(nm, error_procs, opal_namelist_t) { - (void)opal_snprintf_jobid(eps[n].nspace, PMIX_MAX_NSLEN, nm->name.jobid); - eps[n].rank = (int)nm->name.vpid; - ++n; - } - } else { - esz = 0; - eps = NULL; - } - - /* convert the list to an array of pmix_info_t */ - if (NULL != info) { - sz = opal_list_get_size(info); - PMIX_INFO_CREATE(pinfo, sz); - n = 0; - OPAL_LIST_FOREACH(kv, info, opal_value_t) { - (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, kv); - } - } else { - sz = 0; - pinfo = NULL; - } - - /* setup the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); - op->procs = ps; - op->nprocs = psz; - op->error_procs = eps; - op->nerror_procs = esz; - op->info = pinfo; - op->sz = sz; - op->opcbfunc = cbfunc; - op->cbdata = cbdata; - - rc = pmix1_convert_opalrc(status); - rc = PMIx_Notify_error(rc, ps, psz, eps, esz, - pinfo, sz, opcbfunc, op); - if (PMIX_SUCCESS != rc) { - OBJ_RELEASE(op); - } - return pmix1_convert_rc(rc); -} diff --git a/opal/mca/pmix/pmix114/pmix_pmix1.c b/opal/mca/pmix/pmix114/pmix_pmix1.c deleted file mode 100644 index 4ff313fdad..0000000000 --- a/opal/mca/pmix/pmix114/pmix_pmix1.c +++ /dev/null @@ -1,555 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2014 Mellanox Technologies, Inc. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" -#include "opal/constants.h" -#include "opal/types.h" - -#ifdef HAVE_STRING_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif - -#include "opal/dss/dss.h" -#include "opal/mca/event/event.h" -#include "opal/mca/hwloc/base/base.h" -#include "opal/runtime/opal.h" -#include "opal/runtime/opal_progress_threads.h" -#include "opal/util/argv.h" -#include "opal/util/error.h" -#include "opal/util/output.h" -#include "opal/util/proc.h" -#include "opal/util/show_help.h" - -#include "pmix1.h" -#include "opal/mca/pmix/base/base.h" -#include "opal/mca/pmix/pmix_types.h" - -#include "opal/mca/pmix/pmix114/pmix/include/pmix/pmix_common.h" - -/**** C.O.M.M.O.N I.N.T.E.R.F.A.C.E.S ****/ - -/* These are functions used by both client and server to - * access common functions in the embedded PMIx library */ - -static const char *pmix1_get_nspace(opal_jobid_t jobid); -static void pmix1_register_jobid(opal_jobid_t jobid, const char *nspace); - -const opal_pmix_base_module_t opal_pmix_pmix114_module = { - /* client APIs */ - .init = pmix1_client_init, - .finalize = pmix1_client_finalize, - .initialized = pmix1_initialized, - .abort = pmix1_abort, - .commit = pmix1_commit, - .fence = pmix1_fence, - .fence_nb = pmix1_fencenb, - .put = pmix1_put, - .get = pmix1_get, - .get_nb = pmix1_getnb, - .publish = pmix1_publish, - .publish_nb = pmix1_publishnb, - .lookup = pmix1_lookup, - .lookup_nb = pmix1_lookupnb, - .unpublish = pmix1_unpublish, - .unpublish_nb = pmix1_unpublishnb, - .spawn = pmix1_spawn, - .spawn_nb = pmix1_spawnnb, - .connect = pmix1_connect, - .connect_nb = pmix1_connectnb, - .disconnect = pmix1_disconnect, - .disconnect_nb = pmix1_disconnectnb, - .resolve_peers = pmix1_resolve_peers, - .resolve_nodes = pmix1_resolve_nodes, - /* server APIs */ - .server_init = pmix1_server_init, - .server_finalize = pmix1_server_finalize, - .generate_regex = pmix1_server_gen_regex, - .generate_ppn = pmix1_server_gen_ppn, - .server_register_nspace = pmix1_server_register_nspace, - .server_deregister_nspace = pmix1_server_deregister_nspace, - .server_register_client = pmix1_server_register_client, - .server_deregister_client = pmix1_server_deregister_client, - .server_setup_fork = pmix1_server_setup_fork, - .server_dmodex_request = pmix1_server_dmodex, - .server_notify_error = pmix1_server_notify_error, - /* utility APIs */ - .get_version = PMIx_Get_version, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, - .store_local = pmix1_store_local, - .get_nspace = pmix1_get_nspace, - .register_jobid = pmix1_register_jobid -}; - -static const char *pmix1_get_nspace(opal_jobid_t jobid) -{ - opal_pmix1_jobid_trkr_t *jptr; - - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { - if (jptr->jobid == jobid) { - return jptr->nspace; - } - } - return NULL; -} - -static void pmix1_register_jobid(opal_jobid_t jobid, const char *nspace) -{ - opal_pmix1_jobid_trkr_t *jptr; - - /* if we don't already have it, add this to our jobid tracker */ - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { - if (jptr->jobid == jobid) { - return; - } - } - jptr = OBJ_NEW(opal_pmix1_jobid_trkr_t); - (void)strncpy(jptr->nspace, nspace, PMIX_MAX_NSLEN); - jptr->jobid = jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &jptr->super); -} - -pmix_status_t pmix1_convert_opalrc(int rc) -{ - switch (rc) { - case OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; - case OPAL_ERR_COMM_FAILURE: - return PMIX_ERR_COMM_FAILURE; - case OPAL_ERR_NOT_IMPLEMENTED: - return PMIX_ERR_NOT_IMPLEMENTED; - case OPAL_ERR_NOT_SUPPORTED: - return PMIX_ERR_NOT_SUPPORTED; - case OPAL_ERR_NOT_FOUND: - return PMIX_ERR_NOT_FOUND; - case OPAL_ERR_SERVER_NOT_AVAIL: - return PMIX_ERR_SERVER_NOT_AVAIL; - - case OPAL_ERR_BAD_PARAM: - return PMIX_ERR_BAD_PARAM; - case OPAL_ERR_OUT_OF_RESOURCE: - return PMIX_ERR_NOMEM; - - case OPAL_ERR_DATA_VALUE_NOT_FOUND: - return PMIX_ERR_DATA_VALUE_NOT_FOUND; - case OPAL_ERR_IN_ERRNO: - return PMIX_ERR_IN_ERRNO; - case OPAL_ERR_UNREACH: - return PMIX_ERR_UNREACH; - case OPAL_ERR_TIMEOUT: - return PMIX_ERR_TIMEOUT; - case OPAL_ERR_PERM: - return PMIX_ERR_NO_PERMISSIONS; - case OPAL_ERR_PACK_MISMATCH: - return PMIX_ERR_PACK_MISMATCH; - case OPAL_ERR_PACK_FAILURE: - return PMIX_ERR_PACK_FAILURE; - - case OPAL_ERR_UNPACK_FAILURE: - return PMIX_ERR_UNPACK_FAILURE; - case OPAL_ERR_UNPACK_INADEQUATE_SPACE: - return PMIX_ERR_UNPACK_INADEQUATE_SPACE; - case OPAL_ERR_TYPE_MISMATCH: - return PMIX_ERR_TYPE_MISMATCH; - case OPAL_ERR_PROC_ENTRY_NOT_FOUND: - return PMIX_ERR_PROC_ENTRY_NOT_FOUND; - case OPAL_ERR_UNKNOWN_DATA_TYPE: - return PMIX_ERR_UNKNOWN_DATA_TYPE; - case OPAL_ERR_WOULD_BLOCK: - return PMIX_ERR_WOULD_BLOCK; - case OPAL_EXISTS: - return PMIX_EXISTS; - - case OPAL_ERR_SILENT: - return PMIX_ERR_SILENT; - case OPAL_ERROR: - return PMIX_ERROR; - case OPAL_SUCCESS: - return PMIX_SUCCESS; - default: - return PMIX_ERROR; - } -} - -int pmix1_convert_rc(pmix_status_t rc) -{ - switch (rc) { - case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - return OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER; - case PMIX_ERR_COMM_FAILURE: - return OPAL_ERR_COMM_FAILURE; - case PMIX_ERR_NOT_IMPLEMENTED: - return OPAL_ERR_NOT_IMPLEMENTED; - case PMIX_ERR_NOT_SUPPORTED: - return OPAL_ERR_NOT_SUPPORTED; - case PMIX_ERR_NOT_FOUND: - return OPAL_ERR_NOT_FOUND; - case PMIX_ERR_SERVER_NOT_AVAIL: - return OPAL_ERR_SERVER_NOT_AVAIL; - - case PMIX_ERR_INVALID_NAMESPACE: - case PMIX_ERR_INVALID_SIZE: - case PMIX_ERR_INVALID_KEYVALP: - case PMIX_ERR_INVALID_NUM_PARSED: - case PMIX_ERR_INVALID_ARGS: - case PMIX_ERR_INVALID_NUM_ARGS: - case PMIX_ERR_INVALID_LENGTH: - case PMIX_ERR_INVALID_VAL_LENGTH: - case PMIX_ERR_INVALID_VAL: - case PMIX_ERR_INVALID_KEY_LENGTH: - case PMIX_ERR_INVALID_KEY: - case PMIX_ERR_INVALID_ARG: - return OPAL_ERR_BAD_PARAM; - case PMIX_ERR_NOMEM: - return OPAL_ERR_OUT_OF_RESOURCE; - case PMIX_ERR_INIT: - return OPAL_ERROR; - - case PMIX_ERR_DATA_VALUE_NOT_FOUND: - return OPAL_ERR_DATA_VALUE_NOT_FOUND; - case PMIX_ERR_OUT_OF_RESOURCE: - return OPAL_ERR_OUT_OF_RESOURCE; - case PMIX_ERR_RESOURCE_BUSY: - return OPAL_ERR_TEMP_OUT_OF_RESOURCE; - case PMIX_ERR_BAD_PARAM: - return OPAL_ERR_BAD_PARAM; - case PMIX_ERR_IN_ERRNO: - return OPAL_ERR_IN_ERRNO; - case PMIX_ERR_UNREACH: - return OPAL_ERR_UNREACH; - case PMIX_ERR_TIMEOUT: - return OPAL_ERR_TIMEOUT; - case PMIX_ERR_NO_PERMISSIONS: - return OPAL_ERR_PERM; - case PMIX_ERR_PACK_MISMATCH: - return OPAL_ERR_PACK_MISMATCH; - case PMIX_ERR_PACK_FAILURE: - return OPAL_ERR_PACK_FAILURE; - - case PMIX_ERR_UNPACK_FAILURE: - return OPAL_ERR_UNPACK_FAILURE; - case PMIX_ERR_UNPACK_INADEQUATE_SPACE: - return OPAL_ERR_UNPACK_INADEQUATE_SPACE; - case PMIX_ERR_TYPE_MISMATCH: - return OPAL_ERR_TYPE_MISMATCH; - case PMIX_ERR_PROC_ENTRY_NOT_FOUND: - return OPAL_ERR_PROC_ENTRY_NOT_FOUND; - case PMIX_ERR_UNKNOWN_DATA_TYPE: - return OPAL_ERR_UNKNOWN_DATA_TYPE; - case PMIX_ERR_WOULD_BLOCK: - return OPAL_ERR_WOULD_BLOCK; - case PMIX_ERR_READY_FOR_HANDSHAKE: - case PMIX_ERR_HANDSHAKE_FAILED: - case PMIX_ERR_INVALID_CRED: - return OPAL_ERR_COMM_FAILURE; - case PMIX_EXISTS: - return OPAL_EXISTS; - - case PMIX_ERR_SILENT: - return OPAL_ERR_SILENT; - case PMIX_ERROR: - return OPAL_ERROR; - case PMIX_SUCCESS: - return OPAL_SUCCESS; - default: - return OPAL_ERROR; - } -} - -pmix_scope_t pmix1_convert_opalscope(opal_pmix_scope_t scope) { - switch(scope) { - case OPAL_PMIX_LOCAL: - return PMIX_LOCAL; - case OPAL_PMIX_REMOTE: - return PMIX_REMOTE; - case OPAL_PMIX_GLOBAL: - return PMIX_GLOBAL; - default: - return PMIX_SCOPE_UNDEF; - } -} - -void pmix1_value_load(pmix_value_t *v, - opal_value_t *kv) -{ - switch(kv->type) { - case OPAL_UNDEF: - v->type = PMIX_UNDEF; - opal_output(0, "TYPE WAS UNDEF"); - break; - case OPAL_BOOL: - v->type = PMIX_BOOL; - memcpy(&(v->data.flag), &kv->data.flag, 1); - break; - case OPAL_BYTE: - v->type = PMIX_BYTE; - memcpy(&(v->data.byte), &kv->data.byte, 1); - break; - case OPAL_STRING: - v->type = PMIX_STRING; - if (NULL != kv->data.string) { - v->data.string = strdup(kv->data.string); - } else { - v->data.string = NULL; - } - break; - case OPAL_SIZE: - v->type = PMIX_SIZE; - v->data.size = (size_t)kv->data.size; - break; - case OPAL_PID: - v->type = PMIX_PID; - memcpy(&(v->data.pid), &kv->data.pid, sizeof(pid_t)); - break; - case OPAL_INT: - v->type = PMIX_INT; - memcpy(&(v->data.integer), &kv->data.integer, sizeof(int)); - break; - case OPAL_INT8: - v->type = PMIX_INT8; - memcpy(&(v->data.int8), &kv->data.int8, 1); - break; - case OPAL_INT16: - v->type = PMIX_INT16; - memcpy(&(v->data.int16), &kv->data.int16, 2); - break; - case OPAL_INT32: - v->type = PMIX_INT32; - memcpy(&(v->data.int32), &kv->data.int32, 4); - break; - case OPAL_INT64: - v->type = PMIX_INT64; - memcpy(&(v->data.int64), &kv->data.int64, 8); - break; - case OPAL_UINT: - v->type = PMIX_UINT; - memcpy(&(v->data.uint), &kv->data.uint, sizeof(int)); - break; - case OPAL_UINT8: - v->type = PMIX_UINT8; - memcpy(&(v->data.uint8), &kv->data.uint8, 1); - break; - case OPAL_UINT16: - v->type = PMIX_UINT16; - memcpy(&(v->data.uint16), &kv->data.uint16, 2); - break; - case OPAL_UINT32: - v->type = PMIX_UINT32; - memcpy(&(v->data.uint32), &kv->data.uint32, 4); - break; - case OPAL_UINT64: - v->type = PMIX_UINT64; - memcpy(&(v->data.uint64), &kv->data.uint64, 8); - break; - case OPAL_FLOAT: - v->type = PMIX_FLOAT; - memcpy(&(v->data.fval), &kv->data.fval, sizeof(float)); - break; - case OPAL_DOUBLE: - v->type = PMIX_DOUBLE; - memcpy(&(v->data.dval), &kv->data.dval, sizeof(double)); - break; - case OPAL_TIMEVAL: - v->type = PMIX_TIMEVAL; - memcpy(&(v->data.tv), &kv->data.tv, sizeof(struct timeval)); - break; - case OPAL_BYTE_OBJECT: - v->type = PMIX_BYTE_OBJECT; - if (NULL != kv->data.bo.bytes) { - v->data.bo.bytes = (char*)malloc(kv->data.bo.size); - memcpy(v->data.bo.bytes, kv->data.bo.bytes, kv->data.bo.size); - v->data.bo.size = (size_t)kv->data.bo.size; - } else { - v->data.bo.bytes = NULL; - v->data.bo.size = 0; - } - break; - default: - /* silence warnings */ - break; - } -} - -int pmix1_value_unload(opal_value_t *kv, - const pmix_value_t *v) -{ - int rc=OPAL_SUCCESS; - - - switch(v->type) { - case PMIX_UNDEF: - rc = OPAL_ERR_UNKNOWN_DATA_TYPE; - break; - case PMIX_BOOL: - kv->type = OPAL_BOOL; - memcpy(&kv->data.flag, &(v->data.flag), 1); - break; - case PMIX_BYTE: - kv->type = OPAL_BYTE; - memcpy(&kv->data.byte, &(v->data.byte), 1); - break; - case PMIX_STRING: - kv->type = OPAL_STRING; - if (NULL != v->data.string) { - kv->data.string = strdup(v->data.string); - } - break; - case PMIX_SIZE: - kv->type = OPAL_SIZE; - kv->data.size = (int)v->data.size; - break; - case PMIX_PID: - kv->type = OPAL_PID; - memcpy(&kv->data.pid, &(v->data.pid), sizeof(pid_t)); - break; - case PMIX_INT: - kv->type = OPAL_INT; - memcpy(&kv->data.integer, &(v->data.integer), sizeof(int)); - break; - case PMIX_INT8: - kv->type = OPAL_INT8; - memcpy(&kv->data.int8, &(v->data.int8), 1); - break; - case PMIX_INT16: - kv->type = OPAL_INT16; - memcpy(&kv->data.int16, &(v->data.int16), 2); - break; - case PMIX_INT32: - kv->type = OPAL_INT32; - memcpy(&kv->data.int32, &(v->data.int32), 4); - break; - case PMIX_INT64: - kv->type = OPAL_INT64; - memcpy(&kv->data, &(v->data.int64), 8); - break; - case PMIX_UINT: - kv->type = OPAL_UINT; - memcpy(&kv->data, &(v->data.uint), sizeof(int)); - break; - case PMIX_UINT8: - kv->type = OPAL_UINT8; - memcpy(&kv->data, &(v->data.uint8), 1); - break; - case PMIX_UINT16: - kv->type = OPAL_UINT16; - memcpy(&kv->data, &(v->data.uint16), 2); - break; - case PMIX_UINT32: - kv->type = OPAL_UINT32; - memcpy(&kv->data, &(v->data.uint32), 4); - break; - case PMIX_UINT64: - kv->type = OPAL_UINT64; - memcpy(&kv->data, &(v->data.uint64), 8); - break; - case PMIX_FLOAT: - kv->type = OPAL_FLOAT; - memcpy(&kv->data, &(v->data.fval), sizeof(float)); - break; - case PMIX_DOUBLE: - kv->type = OPAL_DOUBLE; - memcpy(&kv->data, &(v->data.dval), sizeof(double)); - break; - case PMIX_TIMEVAL: - kv->type = OPAL_TIMEVAL; - memcpy(&kv->data, &(v->data.tv), sizeof(struct timeval)); - break; - case PMIX_BYTE_OBJECT: - kv->type = OPAL_BYTE_OBJECT; - if (NULL != v->data.bo.bytes && 0 < v->data.bo.size) { - kv->data.bo.bytes = (uint8_t*)malloc(v->data.bo.size); - memcpy(kv->data.bo.bytes, v->data.bo.bytes, v->data.bo.size); - kv->data.bo.size = (int)v->data.bo.size; - } else { - kv->data.bo.bytes = NULL; - kv->data.bo.size = 0; - } - break; - default: - /* silence warnings */ - rc = OPAL_ERROR; - break; - } - return rc; -} - - -/**** INSTANTIATE INTERNAL CLASSES ****/ -OBJ_CLASS_INSTANCE(opal_pmix1_jobid_trkr_t, - opal_list_item_t, - NULL, NULL); - -static void opcon(pmix1_opcaddy_t *p) -{ - memset(&p->p, 0, sizeof(pmix_proc_t)); - p->procs = NULL; - p->nprocs = 0; - p->error_procs = NULL; - p->nerror_procs = 0; - p->info = NULL; - p->ninfo = 0; - p->apps = NULL; - p->sz = 0; - p->opcbfunc = NULL; - p->mdxcbfunc = NULL; - p->valcbfunc = NULL; - p->lkcbfunc = NULL; - p->spcbfunc = NULL; - p->cbdata = NULL; -} -static void opdes(pmix1_opcaddy_t *p) -{ - if (NULL != p->procs) { - PMIX_PROC_FREE(p->procs, p->nprocs); - } - if (NULL != p->error_procs) { - PMIX_PROC_FREE(p->error_procs, p->nerror_procs); - } - if (NULL != p->info) { - PMIX_INFO_FREE(p->info, p->sz); - } - if (NULL != p->apps) { - PMIX_APP_FREE(p->apps, p->sz); - } -} -OBJ_CLASS_INSTANCE(pmix1_opcaddy_t, - opal_object_t, - opcon, opdes); - -static void ocadcon(pmix1_opalcaddy_t *p) -{ - OBJ_CONSTRUCT(&p->procs, opal_list_t); - OBJ_CONSTRUCT(&p->info, opal_list_t); - OBJ_CONSTRUCT(&p->apps, opal_list_t); - p->opcbfunc = NULL; - p->dmdxfunc = NULL; - p->mdxcbfunc = NULL; - p->lkupcbfunc = NULL; - p->spwncbfunc = NULL; - p->cbdata = NULL; - p->odmdxfunc = NULL; - p->ocbdata = NULL; -} -static void ocaddes(pmix1_opalcaddy_t *p) -{ - OPAL_LIST_DESTRUCT(&p->procs); - OPAL_LIST_DESTRUCT(&p->info); - OPAL_LIST_DESTRUCT(&p->apps); -} -OBJ_CLASS_INSTANCE(pmix1_opalcaddy_t, - opal_object_t, - ocadcon, ocaddes); diff --git a/opal/mca/pmix/pmix114/pmix_pmix1_component.c b/opal/mca/pmix/pmix114/pmix_pmix1_component.c deleted file mode 100644 index 2647e05188..0000000000 --- a/opal/mca/pmix/pmix114/pmix_pmix1_component.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2014 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - * These symbols are in a file by themselves to provide nice linker - * semantics. Since linkers generally pull in symbols by object - * files, keeping these symbols as the only symbols in this file - * prevents utility programs such as "ompi_info" from having to import - * entire components just to query their version and parameters. - */ - -#include "opal_config.h" - -#include "opal/constants.h" -#include "opal/class/opal_list.h" -#include "opal/util/proc.h" -#include "opal/mca/pmix/pmix.h" -#include "pmix1.h" - -/* - * Public string showing the pmix pmix114 component version number - */ -const char *opal_pmix_pmix114_component_version_string = - "OPAL pmix114 pmix MCA component version " OPAL_VERSION; - -/* - * Local function - */ -static int pmix114_open(void); -static int pmix114_close(void); -static int pmix114_component_query(mca_base_module_t **module, int *priority); - - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ - -mca_pmix_pmix1_component_t mca_pmix_pmix114_component = { - { - /* First, the mca_component_t struct containing meta information - about the component itself */ - - .base_version = { - /* Indicate that we are a pmix v1.1.0 component (which also - implies a specific MCA version) */ - - OPAL_PMIX_BASE_VERSION_2_0_0, - - /* Component name and version */ - - .mca_component_name = "pmix114", - MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, - OPAL_RELEASE_VERSION), - - /* Component open and close functions */ - - .mca_open_component = pmix114_open, - .mca_close_component = pmix114_close, - .mca_query_component = pmix114_component_query, - }, - /* Next the MCA v1.0.0 component meta data */ - .base_data = { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - } - }, - .native_launch = false -}; - -static int pmix114_open(void) -{ - OBJ_CONSTRUCT(&mca_pmix_pmix114_component.jobids, opal_list_t); - return OPAL_SUCCESS; -} - -static int pmix114_close(void) -{ - OPAL_LIST_DESTRUCT(&mca_pmix_pmix114_component.jobids); - return OPAL_SUCCESS; -} - - -static int pmix114_component_query(mca_base_module_t **module, int *priority) -{ - char *t, *id; - - /* see if a PMIx server is present */ - if (NULL != (t = getenv("PMIX_SERVER_URI")) || - NULL != (id = getenv("PMIX_ID"))) { - /* if PMIx is present, then we are a client and need to use it */ - *priority = 80; - } else { - /* we could be a server, so we still need to be considered */ - *priority = 5; - } - *module = (mca_base_module_t *)&opal_pmix_pmix114_module; - return OPAL_SUCCESS; -} diff --git a/opal/mca/pmix/pmix2x/Makefile.am b/opal/mca/pmix/pmix2x/Makefile.am new file mode 100644 index 0000000000..f9692367a1 --- /dev/null +++ b/opal/mca/pmix/pmix2x/Makefile.am @@ -0,0 +1,53 @@ +# +# Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +# Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2015 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +EXTRA_DIST = autogen.subdirs + +SUBDIRS = pmix + +sources = \ + pmix2x.h \ + pmix2x_component.c \ + pmix2x.c \ + pmix2x_client.c \ + pmix2x_server_south.c \ + pmix2x_server_north.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_opal_pmix_pmix2x_DSO +component_noinst = +component_install = mca_pmix_pmix2x.la +else +component_noinst = libmca_pmix_pmix2x.la +component_install = +endif + +mcacomponentdir = $(opallibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_pmix_pmix2x_la_SOURCES = $(sources) +mca_pmix_pmix2x_la_CFLAGS = $(opal_pmix_pmix2x_CFLAGS) +mca_pmix_pmix2x_la_CPPFLAGS = \ + -I$(srcdir)/pmix/include $(opal_pmix_pmix2x_CPPFLAGS) +mca_pmix_pmix2x_la_LDFLAGS = -module -avoid-version $(opal_pmix_pmix2x_LDFLAGS) +mca_pmix_pmix2x_la_LIBADD = $(opal_pmix_pmix2x_LIBS) +mca_pmix_pmix2x_la_DEPENDENCIES = $(mca_pmix_pmix2x_la_LIBADD) + +noinst_LTLIBRARIES = $(component_noinst) +libmca_pmix_pmix2x_la_SOURCES =$(sources) +libmca_pmix_pmix2x_la_CFLAGS = $(opal_pmix_pmix2x_CFLAGS) +libmca_pmix_pmix2x_la_CPPFLAGS = -I$(srcdir)/pmix/include $(opal_pmix_pmix2x_CPPFLAGS) +libmca_pmix_pmix2x_la_LDFLAGS = -module -avoid-version $(opal_pmix_pmix2x_LDFLAGS) +libmca_pmix_pmix2x_la_LIBADD = $(opal_pmix_pmix2x_LIBS) +libmca_pmix_pmix2x_la_DEPENDENCIES = $(mca_pmix_pmix2x_la_LIBADD) diff --git a/opal/mca/pmix/pmix114/autogen.subdirs b/opal/mca/pmix/pmix2x/autogen.subdirs similarity index 100% rename from opal/mca/pmix/pmix114/autogen.subdirs rename to opal/mca/pmix/pmix2x/autogen.subdirs diff --git a/opal/mca/pmix/pmix2x/configure.m4 b/opal/mca/pmix/pmix2x/configure.m4 new file mode 100644 index 0000000000..b027f3c866 --- /dev/null +++ b/opal/mca/pmix/pmix2x/configure.m4 @@ -0,0 +1,75 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2011-2013 Los Alamos National Security, LLC. +# All rights reserved. +# Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2013-2015 Intel, Inc. All rights reserved. +# Copyright (c) 2015 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_pmix_pmix2x_CONFIG([action-if-found], [action-if-not-found]) +# ----------------------------------------------------------- +AC_DEFUN([MCA_opal_pmix_pmix2x_CONFIG],[ + AC_CONFIG_FILES([opal/mca/pmix/pmix2x/Makefile]) + + OPAL_VAR_SCOPE_PUSH([PMIX_VERSION opal_pmix_pmix2x_save_CPPFLAGS opal_pmix_pmix2x_save_LDFLAGS opal_pmix_pmix2x_save_LIBS opal_pmix_pmix2x_basedir opal_pmix_pmix2x_save_cflags]) + + AS_IF([test "$opal_external_pmix_happy" = "yes"], + [AC_MSG_WARN([using an external pmix; disqualifiying this component]) + opal_pmix_pmix2x_happy=0], + [PMIX_VERSION= + opal_pmix_pmix2x_basedir=opal/mca/pmix/pmix2x + + opal_pmix_pmix2x_save_CFLAGS=$CFLAGS + opal_pmix_pmix2x_save_CPPFLAGS=$CPPFLAGS + opal_pmix_pmix2x_save_LDFLAGS=$LDFLAGS + opal_pmix_pmix2x_save_LIBS=$LIBS + + opal_pmix_pmix2x_args="--enable-embedded-mode --with-pmix-symbol-prefix=opal_pmix_pmix2x_ --disable-visibility --with-libevent-header=\\\"opal/mca/event/$opal_event_base_include\\\" --with-hwloc-header=\\\"$opal_hwloc_base_include\\\"" + AS_IF([test "$enable_debug" = "yes"], + [opal_pmix_pmix2x_args="--enable-debug $opal_pmix_pmix2x_args" + CFLAGS="$OPAL_CFLAGS_BEFORE_PICKY $OPAL_VISIBILITY_CFLAGS -g"], + [opal_pmix_pmix2x_args="--disable-debug $opal_pmix_pmix2x_args" + CFLAGS="$OPAL_CFLAGS_BEFORE_PICKY $OPAL_VISIBILITY_CFLAGS"]) + CPPFLAGS="-I$OPAL_TOP_SRCDIR -I$OPAL_TOP_BUILDDIR -I$OPAL_TOP_SRCDIR/opal/include -I$OPAL_TOP_BUILDDIR/opal/include $CPPFLAGS" + + OPAL_CONFIG_SUBDIR([$opal_pmix_pmix2x_basedir/pmix], + [$opal_pmix_pmix2x_args $opal_subdir_args 'CFLAGS=$CFLAGS' 'CPPFLAGS=$CPPFLAGS'], + [opal_pmix_pmix2x_happy=1], [opal_pmix_pmix2x_happy=0]) + + AS_IF([test $opal_pmix_pmix2x_happy -eq 1], + [PMIX_VERSION="internal v`$srcdir/$opal_pmix_pmix2x_basedir/pmix/config/pmix_get_version.sh $srcdir/$opal_pmix_pmix2x_basedir/pmix/VERSION`" + # Build flags for our Makefile.am + opal_pmix_pmix2x_LIBS='$(OPAL_TOP_BUILDDIR)/'"$opal_pmix_pmix2x_basedir"'/pmix/libpmix.la' + opal_pmix_pmix2x_CPPFLAGS='-I$(OPAL_TOP_BUILDDIR)/opal/mca/pmix/pmix2x/pmix/include/pmix -I$(OPAL_TOP_BUILDDIR)/opal/mca/pmix/pmix2x/pmix/include -I$(OPAL_TOP_BUILDDIR)/opal/mca/pmix/pmix2x/pmix -I$(OPAL_TOP_SRCDIR)/opal/mca/pmix/pmix2x/pmix' + AC_SUBST([opal_pmix_pmix2x_LIBS]) + AC_SUBST([opal_pmix_pmix2x_CPPFLAGS])]) + + CFLAGS=$opal_pmix_pmix2x_save_CFLAGS + CPPFLAGS=$opal_pmix_pmix2x_save_CPPFLAGS + LDFLAGS=$opal_pmix_pmix2x_save_LDFLAGS + LIBS=$opal_pmix_pmix2x_save_LIBS + ]) + + AS_IF([test $opal_pmix_pmix2x_happy -eq 1], + [$1], + [$2]) + + OPAL_VAR_SCOPE_POP +])dnl diff --git a/opal/mca/pmix/pmix2x/pmix/AUTHORS b/opal/mca/pmix/pmix2x/pmix/AUTHORS new file mode 100644 index 0000000000..c429d324c0 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/AUTHORS @@ -0,0 +1,30 @@ +PMIx Authors +================ + +The following cumulative list contains the names and GitHub IDs of +all individuals who have committed code to the PMIx repository. + +Email Name Affiliation(s) +------------------------------- --------------------------- ------------------- +alinask Elena Shipunova Mellanox +annu13 Annapurna Dasari Intel +artpol84 Artem Polyakov Mellanox +dsolt Dave Solt IBM +ggouaillardet Gilles Gouaillardet RIST +hjelmn Nathan Hjelm LANL +igor-ivanov Igor Ivanov Mellanox +jladd-mlnx Joshua Ladd Mellanox +jsquyres Jeff Squyres Cisco, IU +nkogteva Nadezhda Kogteva Mellanox +rhc54 Ralph Castain LANL, Cisco, Intel +------------------------------- --------------------------- ------------------- + +Affiliation abbreviations: +-------------------------- +Cisco = Cisco Systems, Inc. +IBM = International Business Machines, Inc. +Intel = Intel, Inc. +IU = Indiana University +LANL = Los Alamos National Laboratory +Mellanox = Mellanox +RIST = Research Organization for Information Science and Technology diff --git a/opal/mca/pmix/pmix114/pmix/INSTALL b/opal/mca/pmix/pmix2x/pmix/INSTALL similarity index 100% rename from opal/mca/pmix/pmix114/pmix/INSTALL rename to opal/mca/pmix/pmix2x/pmix/INSTALL diff --git a/opal/mca/pmix/pmix114/pmix/LICENSE b/opal/mca/pmix/pmix2x/pmix/LICENSE similarity index 100% rename from opal/mca/pmix/pmix114/pmix/LICENSE rename to opal/mca/pmix/pmix2x/pmix/LICENSE diff --git a/opal/mca/pmix/pmix114/pmix/Makefile.am b/opal/mca/pmix/pmix2x/pmix/Makefile.am similarity index 92% rename from opal/mca/pmix/pmix114/pmix/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/Makefile.am index 0c58505007..915193d0fa 100644 --- a/opal/mca/pmix/pmix114/pmix/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2015 Intel, Inc. All rights reserved +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved # $COPYRIGHT$ # # Additional copyrights may follow @@ -57,7 +57,12 @@ include src/usock/Makefile.am include src/client/Makefile.am include src/server/Makefile.am include src/sec/Makefile.am -include src/common/Makefile.am +include src/event/Makefile.am + +if WANT_DSTORE +include src/sm/Makefile.am +include src/dstore/Makefile.am +endif if PMIX_EMBEDDED_MODE noinst_LTLIBRARIES = libpmix.la @@ -87,7 +92,7 @@ nroff: contrib/md2nroff.pl --source=$$source.md; \ done -EXTRA_DIST += README INSTALL VERSION LICENSE autogen.sh \ +EXTRA_DIST += AUTHORS README INSTALL VERSION LICENSE autogen.sh \ config/pmix_get_version.sh $(man_MANS) \ contrib/platform/optimized \ test/test_common.h test/cli_stages.h \ diff --git a/opal/mca/pmix/pmix114/pmix/NEWS b/opal/mca/pmix/pmix2x/pmix/NEWS similarity index 95% rename from opal/mca/pmix/pmix114/pmix/NEWS rename to opal/mca/pmix/pmix2x/pmix/NEWS index 9e05bc788c..eb858a6717 100644 --- a/opal/mca/pmix/pmix114/pmix/NEWS +++ b/opal/mca/pmix/pmix2x/pmix/NEWS @@ -43,9 +43,6 @@ Master (not on release branches yet) - Do not install internal headers unless specifically requested to do so - Add support for multiple calls to Put/Commit -- Silence some "return code unchecked" warnings. Thanks - to Jim Garlick for pointing them out -- Resolve a race condition during register_clients 1.1.3 diff --git a/opal/mca/pmix/pmix114/pmix/README b/opal/mca/pmix/pmix2x/pmix/README similarity index 100% rename from opal/mca/pmix/pmix114/pmix/README rename to opal/mca/pmix/pmix2x/pmix/README diff --git a/opal/mca/pmix/pmix114/pmix/VERSION b/opal/mca/pmix/pmix2x/pmix/VERSION similarity index 94% rename from opal/mca/pmix/pmix114/pmix/VERSION rename to opal/mca/pmix/pmix2x/pmix/VERSION index ccfd47c71b..c585e36d06 100644 --- a/opal/mca/pmix/pmix114/pmix/VERSION +++ b/opal/mca/pmix/pmix2x/pmix/VERSION @@ -3,7 +3,7 @@ # Copyright (c) 2011 NVIDIA Corporation. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. -# Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2015 Intel, Inc. All rights reserved. # This is the VERSION file for PMIx, describing the precise # version of PMIx in this distribution. The various components of @@ -13,9 +13,9 @@ # major, minor, and release are generally combined in the form # ... -major=1 -minor=1 -release=4 +major=2 +minor=0 +release=0 # greek is used for alpha or beta release tags. If it is non-empty, # it will be appended to the version number. It does not have to be @@ -23,14 +23,14 @@ release=4 # The only requirement is that it must be entirely printable ASCII # characters and have no white space. -greek=rc3 +greek=a1 # If repo_rev is empty, then the repository version number will be # obtained during "make dist" via the "git describe --tags --always" # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=git4695e45 +repo_rev=git99b989f # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +44,7 @@ tarball_version= # The date when this release was created -date="Jun 01, 2016" +date="Jun 14, 2016" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library @@ -75,4 +75,4 @@ date="Jun 01, 2016" # Version numbers are described in the Libtool current:revision:age # format. -libpmix_so_version=2:3:0 +libpmix_so_version=0:0:0 diff --git a/opal/mca/pmix/pmix114/pmix/autogen.sh b/opal/mca/pmix/pmix2x/pmix/autogen.sh similarity index 100% rename from opal/mca/pmix/pmix114/pmix/autogen.sh rename to opal/mca/pmix/pmix2x/pmix/autogen.sh diff --git a/opal/mca/pmix/pmix114/pmix/config/Makefile.am b/opal/mca/pmix/pmix2x/pmix/config/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/config/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/config/c_get_alignment.m4 b/opal/mca/pmix/pmix2x/pmix/config/c_get_alignment.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/c_get_alignment.m4 rename to opal/mca/pmix/pmix2x/pmix/config/c_get_alignment.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/distscript.sh b/opal/mca/pmix/pmix2x/pmix/config/distscript.sh similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/distscript.sh rename to opal/mca/pmix/pmix2x/pmix/config/distscript.sh diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 similarity index 98% rename from opal/mca/pmix/pmix114/pmix/config/pmix.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix.m4 index 7b65aefa3b..c406785dbb 100644 --- a/opal/mca/pmix/pmix114/pmix/config/pmix.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 @@ -753,6 +753,26 @@ AC_DEFINE_UNQUOTED([PMIX_WANT_PRETTY_PRINT_STACKTRACE], [$WANT_PRETTY_PRINT_STACKTRACE], [if want pretty-print stack trace feature]) +# +# Do we want the shared memory datastore usage? +# + +AC_MSG_CHECKING([if want special dstore usage]) +AC_ARG_ENABLE([dstore], + [AC_HELP_STRING([--enable-dstore], + [Using special datastore (default: disabled)])]) +if test "$enable_dstore" = "yes" ; then + AC_MSG_RESULT([yes]) + WANT_DSTORE=1 +else + AC_MSG_RESULT([no]) + WANT_DSTORE=0 +fi +AC_DEFINE_UNQUOTED([PMIX_ENABLE_DSTORE], + [$WANT_DSTORE], + [if want special dstore feature]) +AM_CONDITIONAL([WANT_DSTORE],[test "x$enable_dstore" = "xyes"]) + # # Ident string # diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_attributes.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_attributes.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_attributes.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_attributes.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_broken_qsort.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_broken_qsort.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_broken_qsort.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_broken_qsort.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_compiler_version.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_compiler_version.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_compiler_version.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_compiler_version.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_icc.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_icc.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_icc.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_icc.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_ident.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_ident.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_ident.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_ident.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_munge.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_munge.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_munge.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_munge.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_package.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_package.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_package.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_package.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_sasl.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_sasl.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_sasl.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_sasl.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_vendor.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_vendor.m4 similarity index 99% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_vendor.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_vendor.m4 index ba3f1a5a8d..cf96c46256 100644 --- a/opal/mca/pmix/pmix114/pmix/config/pmix_check_vendor.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_vendor.m4 @@ -95,7 +95,7 @@ AC_DEFUN([_PMIX_CHECK_COMPILER_VENDOR], [ # Fujitsu AS_IF([test "$pmix_check_compiler_vendor_result" = "unknown"], - [PMIX_IF_IFELSE([defined(__FUJITSU)], + [PMIX_IF_IFELSE([defined(__FUJITSU)], [pmix_check_compiler_vendor_result="fujitsu"])]) # GNU diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_check_visibility.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_visibility.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_check_visibility.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_check_visibility.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_ensure_contains_optflags.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_ensure_contains_optflags.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_ensure_contains_optflags.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_ensure_contains_optflags.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_functions.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_functions.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_functions.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_functions.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_get_version.sh b/opal/mca/pmix/pmix2x/pmix/config/pmix_get_version.sh similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_get_version.sh rename to opal/mca/pmix/pmix2x/pmix/config/pmix_get_version.sh diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_load_platform.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_load_platform.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_load_platform.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_load_platform.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_search_libs.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_search_libs.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_search_libs.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_search_libs.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_setup_cc.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cc.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_setup_cc.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cc.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_setup_hwloc.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_hwloc.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_setup_hwloc.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_setup_hwloc.m4 diff --git a/opal/mca/pmix/pmix114/pmix/config/pmix_setup_libevent.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_libevent.m4 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/config/pmix_setup_libevent.m4 rename to opal/mca/pmix/pmix2x/pmix/config/pmix_setup_libevent.m4 diff --git a/opal/mca/pmix/pmix114/pmix/configure.ac b/opal/mca/pmix/pmix2x/pmix/configure.ac similarity index 100% rename from opal/mca/pmix/pmix114/pmix/configure.ac rename to opal/mca/pmix/pmix2x/pmix/configure.ac diff --git a/opal/mca/pmix/pmix114/pmix/contrib/platform/optimized b/opal/mca/pmix/pmix2x/pmix/contrib/platform/optimized similarity index 100% rename from opal/mca/pmix/pmix114/pmix/contrib/platform/optimized rename to opal/mca/pmix/pmix2x/pmix/contrib/platform/optimized diff --git a/opal/mca/pmix/pmix114/pmix/contrib/pmix-valgrind.supp b/opal/mca/pmix/pmix2x/pmix/contrib/pmix-valgrind.supp similarity index 99% rename from opal/mca/pmix/pmix114/pmix/contrib/pmix-valgrind.supp rename to opal/mca/pmix/pmix2x/pmix/contrib/pmix-valgrind.supp index 94651905c7..4de2cf66b3 100644 --- a/opal/mca/pmix/pmix114/pmix/contrib/pmix-valgrind.supp +++ b/opal/mca/pmix/pmix2x/pmix/contrib/pmix-valgrind.supp @@ -2,9 +2,9 @@ # # Copyright (c) 2015 Intel, Inc. All rights reserved. # $COPYRIGHT$ -# +# # Additional copyrights may follow -# +# # $HEADER$ # diff --git a/opal/mca/pmix/pmix114/pmix/examples/Makefile.am b/opal/mca/pmix/pmix2x/pmix/examples/Makefile.am similarity index 90% rename from opal/mca/pmix/pmix114/pmix/examples/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/examples/Makefile.am index 886dd37543..5cc39a67b9 100644 --- a/opal/mca/pmix/pmix114/pmix/examples/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/examples/Makefile.am @@ -21,7 +21,7 @@ AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/src/api -noinst_PROGRAMS = client dmodex dynamic fault pub +noinst_PROGRAMS = client dmodex dynamic fault pub server client_SOURCES = client.c client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) @@ -43,5 +43,9 @@ pub_SOURCES = pub.c pub_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pub_LDADD = $(top_builddir)/libpmix.la +server_SOURCES = pub.c +server_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +server_LDADD = $(top_builddir)/libpmix.la + distclean-local: rm -f *.o client dmodex dynamic fault pub server diff --git a/opal/mca/pmix/pmix114/pmix/examples/client.c b/opal/mca/pmix/pmix2x/pmix/examples/client.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/examples/client.c rename to opal/mca/pmix/pmix2x/pmix/examples/client.c index 59add5da3a..cf1dbe0f09 100644 --- a/opal/mca/pmix/pmix114/pmix/examples/client.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/client.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -45,7 +45,7 @@ int main(int argc, char **argv) bool flag; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } @@ -163,7 +163,7 @@ int main(int argc, char **argv) done: /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/examples/dmodex.c b/opal/mca/pmix/pmix2x/pmix/examples/dmodex.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/examples/dmodex.c rename to opal/mca/pmix/pmix2x/pmix/examples/dmodex.c index 23f224a1b9..714f3d3df1 100644 --- a/opal/mca/pmix/pmix114/pmix/examples/dmodex.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/dmodex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -101,7 +101,7 @@ int main(int argc, char **argv) bool active; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } @@ -221,7 +221,7 @@ int main(int argc, char **argv) done: /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/examples/dynamic.c b/opal/mca/pmix/pmix2x/pmix/examples/dynamic.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/examples/dynamic.c rename to opal/mca/pmix/pmix2x/pmix/examples/dynamic.c index 195c130e62..22f0c1ccf0 100644 --- a/opal/mca/pmix/pmix114/pmix/examples/dynamic.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/dynamic.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -58,7 +58,7 @@ int main(int argc, char **argv) } /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } @@ -189,7 +189,7 @@ int main(int argc, char **argv) /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/examples/fault.c b/opal/mca/pmix/pmix2x/pmix/examples/fault.c similarity index 77% rename from opal/mca/pmix/pmix114/pmix/examples/fault.c rename to opal/mca/pmix/pmix2x/pmix/examples/fault.c index 11bc611df5..7a103524d5 100644 --- a/opal/mca/pmix/pmix114/pmix/examples/fault.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/fault.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -34,9 +34,12 @@ static pmix_proc_t myproc; static bool completed; -static void notification_fn(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) +static void notification_fn(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) { fprintf(stderr, "Client %s:%d NOTIFIED with status %d\n", myproc.nspace, myproc.rank, status); completed = true; @@ -48,12 +51,12 @@ static void op_callbk(pmix_status_t status, fprintf(stderr, "client: OP CALLBACK CALLED WITH STATUS %d", status); } -static void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, - void *cbdata) +static void errhandler_reg_callbk(pmix_status_t status, + size_t errhandler_ref, + void *cbdata) { - fprintf(stderr, "cleint: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%d", - status, errhandler_ref); + fprintf(stderr, "client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref); } int main(int argc, char **argv) @@ -65,7 +68,7 @@ int main(int argc, char **argv) uint32_t nprocs; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } @@ -82,7 +85,8 @@ int main(int argc, char **argv) completed = false; /* register our errhandler */ - PMIx_Register_errhandler(NULL, 0, notification_fn, errhandler_reg_callbk, NULL); + PMIx_Register_event_handler(NULL, 0, NULL, 0, + notification_fn, errhandler_reg_callbk, NULL); /* call fence to sync */ PMIX_PROC_CONSTRUCT(&proc); @@ -110,9 +114,9 @@ int main(int argc, char **argv) done: /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); - PMIx_Deregister_errhandler(0, op_callbk, NULL); + PMIx_Deregister_event_handler(1, op_callbk, NULL); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/examples/pub.c b/opal/mca/pmix/pmix2x/pmix/examples/pub.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/examples/pub.c rename to opal/mca/pmix/pmix2x/pmix/examples/pub.c index 482a363bbf..65fc528b19 100644 --- a/opal/mca/pmix/pmix114/pmix/examples/pub.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/pub.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -43,7 +43,7 @@ int main(int argc, char **argv) pmix_pdata_t *pdata; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } @@ -154,7 +154,7 @@ int main(int argc, char **argv) done: /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/include/Makefile.am b/opal/mca/pmix/pmix2x/pmix/include/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/include/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/include/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/include/pmi.h b/opal/mca/pmix/pmix2x/pmix/include/pmi.h similarity index 99% rename from opal/mca/pmix/pmix114/pmix/include/pmi.h rename to opal/mca/pmix/pmix2x/pmix/include/pmi.h index bc09ca1ed9..328eee1175 100644 --- a/opal/mca/pmix/pmix114/pmix/include/pmi.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmi.h @@ -220,7 +220,7 @@ Return values: int PMI_Get_appnum( int *appnum ); /*@ -PMI_Publish_name - publish a name +PMI_Publish_name - publish a name Input parameters: . service_name - string representing the service being published @@ -444,7 +444,7 @@ This function returns the string length required to store a keyval space name. A routine is used rather than setting a maximum value in 'pmi.h' to allow different implementations of PMI to be used with the same executable. These -different implementations may allow different maximum lengths; by using a +different implementations may allow different maximum lengths; by using a routine here, we can interface with a variety of implementations of PMI. @*/ @@ -544,7 +544,7 @@ Return values: Notes: This function puts the key/value pair in the specified keyval space. The -value is not visible to other processes until 'PMI_KVS_Commit()' is called. +value is not visible to other processes until 'PMI_KVS_Commit()' is called. The function may complete locally. After 'PMI_KVS_Commit()' is called, the value may be retrieved by calling 'PMI_KVS_Get()'. All keys put to a keyval space must be unique to the keyval space. You may not put more than once @@ -649,7 +649,7 @@ Return values: - PMI_FAIL - failed to get the next keyval pair Notes: -This function retrieves the next keyval pair from the specified keyval space. +This function retrieves the next keyval pair from the specified keyval space. 'PMI_KVS_Iter_first()' must have been previously called. The end of the keyval space is specified by returning an empty key string. The output parameters, key and val, must be at least as long as the values returned by @@ -682,7 +682,7 @@ Input Parameters: . cmds - array of command strings . argvs - array of argv arrays for each command string . maxprocs - array of maximum processes to spawn for each command string -. info_keyval_sizes - array giving the number of elements in each of the +. info_keyval_sizes - array giving the number of elements in each of the 'info_keyval_vectors' . info_keyval_vectors - array of keyval vector arrays . preput_keyval_size - Number of elements in 'preput_keyval_vector' @@ -703,7 +703,7 @@ field refers to the size of the array parameters - 'cmd', 'argvs', 'maxprocs', to the size of the 'preput_keyval_vector' array. The 'preput_keyval_vector' contains keyval pairs that will be put in the keyval space of the newly created process group before the processes are started. The 'maxprocs' array -specifies the desired number of processes to create for each 'cmd' string. +specifies the desired number of processes to create for each 'cmd' string. The actual number of processes may be less than the numbers specified in maxprocs. The acceptable number of processes spawned may be controlled by ``soft'' keyvals in the info arrays. The ``soft'' option is specified by @@ -774,7 +774,7 @@ Notes: This function removes PMI specific arguments from the command line and creates the corresponding 'PMI_keyval_t' structures for them. It returns an array and size to the caller that can then be passed to 'PMI_Spawn_multiple()'. -The array can be freed by 'PMI_Free_keyvals()'. The routine 'free()' should +The array can be freed by 'PMI_Free_keyvals()'. The routine 'free()' should not be used to free this array as there is no requirement that the array be allocated with 'malloc()'. @@ -795,7 +795,7 @@ Return values: Notes: This function frees the data returned by 'PMI_Args_to_keyval' and 'PMI_Parse_option'. - Using this routine instead of 'free' allows the PMI package to track + Using this routine instead of 'free' allows the PMI package to track allocation of storage or to use interal storage as it sees fit. @*/ int PMI_Free_keyvals(PMI_keyval_t keyvalp[], int size); diff --git a/opal/mca/pmix/pmix114/pmix/include/pmi2.h b/opal/mca/pmix/pmix2x/pmix/include/pmi2.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/include/pmi2.h rename to opal/mca/pmix/pmix2x/pmix/include/pmi2.h diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix.h b/opal/mca/pmix/pmix2x/pmix/include/pmix.h similarity index 83% rename from opal/mca/pmix/pmix114/pmix/include/pmix.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix.h index 5c879eacc2..545429251d 100644 --- a/opal/mca/pmix/pmix114/pmix/include/pmix.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix.h @@ -76,19 +76,34 @@ BEGIN_C_DECLS * * Note that the PMIx client library is referenced counted, and so multiple * calls to PMIx_Init are allowed. Thus, one way to obtain the namespace and - * rank of the process is to simply call PMIx_Init with a non-NULL parameter. */ -pmix_status_t PMIx_Init(pmix_proc_t *proc); + * rank of the process is to simply call PMIx_Init with a non-NULL parameter. + * + * The info array is used to pass user requests pertaining to the init + * and subsequent operations. Pass a _NULL_ value for the array pointer + * is supported if no directives are desired. + */ + pmix_status_t PMIx_Init(pmix_proc_t *proc, + pmix_info_t info[], size_t ninfo); /* Finalize the PMIx client, closing the connection to the local server. * An error code will be returned if, for some reason, the connection - * cannot be closed. */ -pmix_status_t PMIx_Finalize(void); + * cannot be closed. + * + * The info array is used to pass user requests regarding the finalize + * operation. This can include: + * + * (a) PMIX_EMBED_BARRIER - By default, PMIx_Finalize does not include an + * internal barrier operation. This attribute directs PMIx_Finalize to + * execute a barrier as part of the finalize operation. + */ + pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo); + /* Returns _true_ if the PMIx client has been successfully initialized, * returns _false_ otherwise. Note that the function only reports the * internal state of the PMIx client - it does not verify an active * connection with the server, nor that the server is functional. */ -int PMIx_Initialized(void); + int PMIx_Initialized(void); /* Request that the provided array of procs be aborted, returning the @@ -108,22 +123,22 @@ int PMIx_Initialized(void); * caused by multiple processes calling PMIx_Abort are left to the * server implementation to resolve with regard to which status is * returned and what messages (if any) are printed. */ -pmix_status_t PMIx_Abort(int status, const char msg[], - pmix_proc_t procs[], size_t nprocs); + pmix_status_t PMIx_Abort(int status, const char msg[], + pmix_proc_t procs[], size_t nprocs); /* Push a value into the client's namespace. The client library will cache * the information locally until _PMIx_Commit_ is called. The provided scope * value is passed to the local PMIx server, which will distribute the data * as directed. */ -pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_value_t *val); + pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_value_t *val); /* Push all previously _PMIx_Put_ values to the local PMIx server. * This is an asynchronous operation - the library will immediately * return to the caller while the data is transmitted to the local * server in the background */ -pmix_status_t PMIx_Commit(void); + pmix_status_t PMIx_Commit(void); /* Execute a blocking barrier across the processes identified in the @@ -161,14 +176,14 @@ pmix_status_t PMIx_Commit(void); * the timeout parameter can help avoid "hangs" due to programming errors * that prevent one or more procs from reaching the "fence". */ -pmix_status_t PMIx_Fence(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo); + pmix_status_t PMIx_Fence(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo); /* Non-blocking version of PMIx_Fence. Note that the function will return * an error if a _NULL_ callback function is given. */ -pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Retrieve information for the specified _key_ as published by the process @@ -187,17 +202,17 @@ pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs, * an error. The timeout parameter can help avoid "hangs" due to programming * errors that prevent the target proc from ever exposing its data. */ -pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], - const pmix_info_t info[], size_t ninfo, - pmix_value_t **val); + pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], + const pmix_info_t info[], size_t ninfo, + pmix_value_t **val); /* A non-blocking operation version of PMIx_Get - the callback function will * be executed once the specified data has been _PMIx_Put_ * by the identified process and retrieved by the local server. The info * array is used as described above for the blocking form of this call. */ -pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char key[], - const pmix_info_t info[], size_t ninfo, - pmix_value_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char key[], + const pmix_info_t info[], size_t ninfo, + pmix_value_cbfunc_t cbfunc, void *cbdata); /* Publish the data in the info array for lookup. By default, @@ -219,9 +234,9 @@ pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char key[], * return immediately, executing the callback when the server confirms * availability of the data. */ -pmix_status_t PMIx_Publish(const pmix_info_t info[], size_t ninfo); -pmix_status_t PMIx_Publish_nb(const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Publish(const pmix_info_t info[], size_t ninfo); + pmix_status_t PMIx_Publish_nb(const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Lookup information published by this or another process. By default, @@ -260,8 +275,8 @@ pmix_status_t PMIx_Publish_nb(const pmix_info_t info[], size_t ninfo, * (b) PMIX_TIMEOUT - max time to wait for data to become available. * */ -pmix_status_t PMIx_Lookup(pmix_pdata_t data[], size_t ndata, - const pmix_info_t info[], size_t ninfo); + pmix_status_t PMIx_Lookup(pmix_pdata_t data[], size_t ndata, + const pmix_info_t info[], size_t ninfo); /* Non-blocking form of the _PMIx_Lookup_ function. Data for * the provided NULL-terminated keys array will be returned @@ -269,8 +284,8 @@ pmix_status_t PMIx_Lookup(pmix_pdata_t data[], size_t ndata, * behavior is to _not_ wait for data to be published. The * info keys can be used to modify the behavior as previously * described */ -pmix_status_t PMIx_Lookup_nb(char **keys, const pmix_info_t info[], size_t ninfo, - pmix_lookup_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Lookup_nb(char **keys, const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata); /* Unpublish data posted by this process using the given keys. @@ -281,15 +296,15 @@ pmix_status_t PMIx_Lookup_nb(char **keys, const pmix_info_t info[], size_t ninfo * By default, the range is assumed to be PMIX_SESSION. Changes * to the range, and any additional directives, can be provided * in the pmix_info_t array */ -pmix_status_t PMIx_Unpublish(char **keys, - const pmix_info_t info[], size_t ninfo); + pmix_status_t PMIx_Unpublish(char **keys, + const pmix_info_t info[], size_t ninfo); /* Non-blocking form of the _PMIx_Unpublish_ function. The * callback function will be executed once the server confirms * removal of the specified data. */ -pmix_status_t PMIx_Unpublish_nb(char **keys, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Unpublish_nb(char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Spawn a new job. The assigned namespace of the spawned applications @@ -322,17 +337,17 @@ pmix_status_t PMIx_Unpublish_nb(char **keys, * (c) PMIX_NOTIFY_COMPLETION - notify the parent process when the * child job terminates, either normally or with error */ -pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, - const pmix_app_t apps[], size_t napps, - char nspace[]); + pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + char nspace[]); /* Non-blocking form of the _PMIx_Spawn_ function. The callback * will be executed upon launch of the specified applications, * or upon failure to launch any of them. */ -pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t ninfo, - const pmix_app_t apps[], size_t napps, - pmix_spawn_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata); /* Record the specified processes as "connected". Both blocking and non-blocking * versions are provided. This means that the resource manager should treat the @@ -354,12 +369,12 @@ pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t ninfo, * user-level directives regarding the algorithm to be used for the collective * operation involved in the "connect", timeout constraints, and other options * available from the host RM */ -pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo); + pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo); -pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Disconnect a previously connected set of processes. An error will be returned * if the specified set of procs was not previously "connected". As above, a process @@ -367,12 +382,12 @@ pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs, * is not allowed to reconnect to a set of procs that has not fully completed * disconnect - i.e., you have to fully disconnect before you can reconnect to the * _same_ group of processes. The info array is used as above. */ -pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo); + pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo); -pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); + pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Given a node name, return an array of processes within the specified nspace * on that node. If the nspace is NULL, then all processes on the node will @@ -381,15 +396,15 @@ pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t nprocs, * for releasing the array when done with it - the PMIX_PROC_FREE macro is * provided for this purpose. */ -pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *nspace, - pmix_proc_t **procs, size_t *nprocs); + pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *nspace, + pmix_proc_t **procs, size_t *nprocs); /* Given an nspace, return the list of nodes hosting processes within * that nspace. The returned string will contain a comma-delimited list * of nodenames. The caller is responsible for releasing the string * when done with it */ -pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist); + pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist); END_C_DECLS #endif diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_config_bottom.h b/opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_config_bottom.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_config_bottom.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_config_bottom.h diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_config_top.h b/opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_config_top.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_config_top.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_config_top.h diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_stdint.h b/opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_stdint.h similarity index 99% rename from opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_stdint.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_stdint.h index f873381f0c..88cf28b2b4 100644 --- a/opal/mca/pmix/pmix114/pmix/include/pmix/autogen/pmix_stdint.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix/autogen/pmix_stdint.h @@ -5,15 +5,15 @@ * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ * * This file includes the C99 stdint.h file if available, and otherwise @@ -188,7 +188,7 @@ typedef unsigned long long uintptr_t; /* inttypes.h printf specifiers */ #ifdef HAVE_INTTYPES_H # include -#else +#else # if SIZEOF_LONG == 8 # define __PRI64_PREFIX "l" @@ -307,7 +307,7 @@ typedef unsigned long long uintptr_t; # define PRIoPTR __PRIPTR_PREFIX "o" # define PRIuPTR __PRIPTR_PREFIX "u" # define PRIxPTR __PRIPTR_PREFIX "x" -# define PRIXPTR __PRIPTR_PREFIX "X" +# define PRIXPTR __PRIPTR_PREFIX "X" #endif diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix/pmix_common.h b/opal/mca/pmix/pmix2x/pmix/include/pmix/pmix_common.h similarity index 60% rename from opal/mca/pmix/pmix114/pmix/include/pmix/pmix_common.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix/pmix_common.h index 512f4f15b8..5a52b15636 100644 --- a/opal/mca/pmix/pmix114/pmix/include/pmix/pmix_common.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix/pmix_common.h @@ -1,5 +1,6 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -89,35 +90,39 @@ BEGIN_C_DECLS * these keys are RESERVED */ #define PMIX_ATTR_UNDEF NULL +/* initialization attributes */ +#define PMIX_EVENT_BASE "pmix.evbase" // (struct event_base *) pointer to libevent event_base to use in place + // of the internal progress thread + /* identification attributes */ -#define PMIX_USERID "pmix.euid" // (uint32_t) effective user id -#define PMIX_GRPID "pmix.egid" // (uint32_t) effective group id +#define PMIX_USERID "pmix.euid" // (uint32_t) effective user id +#define PMIX_GRPID "pmix.egid" // (uint32_t) effective group id /* attributes for the rendezvous socket */ -#define PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid) +#define PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid) /* general proc-level attributes */ -#define PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch -#define PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc -#define PMIX_SPAWNED "pmix.spawned" // (bool) true if this proc resulted from a call to PMIx_Spawn -#define PMIX_ARCH "pmix.arch" // (uint32_t) datatype architecture flag +#define PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch +#define PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc +#define PMIX_SPAWNED "pmix.spawned" // (bool) true if this proc resulted from a call to PMIx_Spawn +#define PMIX_ARCH "pmix.arch" // (uint32_t) datatype architecture flag /* scratch directory locations for use by applications */ -#define PMIX_TMPDIR "pmix.tmpdir" // (char*) top-level tmp dir assigned to session -#define PMIX_NSDIR "pmix.nsdir" // (char*) sub-tmpdir assigned to namespace -#define PMIX_PROCDIR "pmix.pdir" // (char*) sub-nsdir assigned to proc +#define PMIX_TMPDIR "pmix.tmpdir" // (char*) top-level tmp dir assigned to session +#define PMIX_NSDIR "pmix.nsdir" // (char*) sub-tmpdir assigned to namespace +#define PMIX_PROCDIR "pmix.pdir" // (char*) sub-nsdir assigned to proc /* information about relative ranks as assigned by the RM */ -#define PMIX_JOBID "pmix.jobid" // (char*) jobid assigned by scheduler -#define PMIX_APPNUM "pmix.appnum" // (uint32_t) app number within the job -#define PMIX_RANK "pmix.rank" // (uint32_t) process rank within the job -#define PMIX_GLOBAL_RANK "pmix.grank" // (uint32_t) rank spanning across all jobs in this session -#define PMIX_APP_RANK "pmix.apprank" // (uint32_t) rank within this app -#define PMIX_NPROC_OFFSET "pmix.offset" // (uint32_t) starting global rank of this job -#define PMIX_LOCAL_RANK "pmix.lrank" // (uint16_t) rank on this node within this job -#define PMIX_NODE_RANK "pmix.nrank" // (uint16_t) rank on this node spanning all jobs -#define PMIX_LOCALLDR "pmix.lldr" // (uint64_t) opal_identifier of lowest rank on this node within this job -#define PMIX_APPLDR "pmix.aldr" // (uint32_t) lowest rank in this app within this job +#define PMIX_JOBID "pmix.jobid" // (char*) jobid assigned by scheduler +#define PMIX_APPNUM "pmix.appnum" // (uint32_t) app number within the job +#define PMIX_RANK "pmix.rank" // (uint32_t) process rank within the job +#define PMIX_GLOBAL_RANK "pmix.grank" // (uint32_t) rank spanning across all jobs in this session +#define PMIX_APP_RANK "pmix.apprank" // (uint32_t) rank within this app +#define PMIX_NPROC_OFFSET "pmix.offset" // (uint32_t) starting global rank of this job +#define PMIX_LOCAL_RANK "pmix.lrank" // (uint16_t) rank on this node within this job +#define PMIX_NODE_RANK "pmix.nrank" // (uint16_t) rank on this node spanning all jobs +#define PMIX_LOCALLDR "pmix.lldr" // (uint64_t) opal_identifier of lowest rank on this node within this job +#define PMIX_APPLDR "pmix.aldr" // (uint32_t) lowest rank in this app within this job /* proc location-related info */ /* For PMIX_HOSTNAME, three use-cases exist for PMIx_Get: @@ -133,147 +138,145 @@ BEGIN_C_DECLS * (c) Specifying a namespace and a rank will return the name of the * host that proc is on */ -#define PMIX_HOSTNAME "pmix.hname" // (char*) see above comment -#define PMIX_NODEID "pmix.nodeid" // (uint32_t) node identifier -#define PMIX_LOCAL_PEERS "pmix.lpeers" // (char*) comma-delimited string of ranks on this node within the specified nspace -#define PMIX_LOCAL_CPUSETS "pmix.lcpus" // (char*) colon-delimited cpusets of local peers within the specified nspace -#define PMIX_PROC_URI "pmix.puri" // (char*) URI containing contact info for proc +#define PMIX_HOSTNAME "pmix.hname" // (char*) see above comment +#define PMIX_NODEID "pmix.nodeid" // (uint32_t) node identifier +#define PMIX_LOCAL_PEERS "pmix.lpeers" // (char*) comma-delimited string of ranks on this node within the specified nspace +#define PMIX_LOCAL_CPUSETS "pmix.lcpus" // (char*) colon-delimited cpusets of local peers within the specified nspace +#define PMIX_PROC_URI "pmix.puri" // (char*) URI containing contact info for proc /* size info */ -#define PMIX_UNIV_SIZE "pmix.univ.size" // (uint32_t) #procs in this nspace -#define PMIX_JOB_SIZE "pmix.job.size" // (uint32_t) #procs in this job -#define PMIX_LOCAL_SIZE "pmix.local.size" // (uint32_t) #procs in this job on this node -#define PMIX_NODE_SIZE "pmix.node.size" // (uint32_t) #procs across all jobs on this node -#define PMIX_MAX_PROCS "pmix.max.size" // (uint32_t) max #procs for this job +#define PMIX_UNIV_SIZE "pmix.univ.size" // (uint32_t) #procs in this nspace +#define PMIX_JOB_SIZE "pmix.job.size" // (uint32_t) #procs in this job +#define PMIX_APP_SIZE "pmix.app.size" // (uint32_t) #procs in this application +#define PMIX_LOCAL_SIZE "pmix.local.size" // (uint32_t) #procs in this job on this node +#define PMIX_NODE_SIZE "pmix.node.size" // (uint32_t) #procs across all jobs on this node +#define PMIX_MAX_PROCS "pmix.max.size" // (uint32_t) max #procs for this job /* topology info */ -#define PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology -#define PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology -#define PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job -#define PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object +#define PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology +#define PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology +#define PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job +#define PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object /* request-related info */ -#define PMIX_COLLECT_DATA "pmix.collect" // (bool) collect data and return it at the end of the operation -#define PMIX_TIMEOUT "pmix.timeout" // (int) time in sec before specified operation should time out -#define PMIX_WAIT "pmix.wait" // (int) caller requests that the server wait until at least the specified - // #values are found (0 => all and is the default) -#define PMIX_COLLECTIVE_ALGO "pmix.calgo" // (char*) comma-delimited list of algorithms to use for collective -#define PMIX_COLLECTIVE_ALGO_REQD "pmix.calreqd" // (bool) if true, indicates that the requested choice of algo is mandatory -#define PMIX_NOTIFY_COMPLETION "pmix.notecomp" // (bool) notify parent process upon termination of child job -#define PMIX_RANGE "pmix.range" // (int) pmix_data_range_t value for calls to publish/lookup/unpublish -#define PMIX_PERSISTENCE "pmix.persist" // (int) pmix_persistence_t value for calls to publish -#define PMIX_OPTIONAL "pmix.optional" // (bool) look only in the immediate data store for the requested value - do - // not request data from the server if not found +#define PMIX_COLLECT_DATA "pmix.collect" // (bool) collect data and return it at the end of the operation +#define PMIX_TIMEOUT "pmix.timeout" // (int) time in sec before specified operation should time out +#define PMIX_WAIT "pmix.wait" // (int) caller requests that the server wait until at least the specified + // #values are found (0 => all and is the default) +#define PMIX_COLLECTIVE_ALGO "pmix.calgo" // (char*) comma-delimited list of algorithms to use for collective +#define PMIX_COLLECTIVE_ALGO_REQD "pmix.calreqd" // (bool) if true, indicates that the requested choice of algo is mandatory +#define PMIX_NOTIFY_COMPLETION "pmix.notecomp" // (bool) notify parent process upon termination of child job +#define PMIX_RANGE "pmix.range" // (int) pmix_data_range_t value for calls to publish/lookup/unpublish +#define PMIX_PERSISTENCE "pmix.persist" // (int) pmix_persistence_t value for calls to publish +#define PMIX_OPTIONAL "pmix.optional" // (bool) look only in the immediate data store for the requested value - do + // not request data from the server if not found +#define PMIX_EMBED_BARRIER "pmix.embed.barrier" // (bool) execute a blocking fence operation before executing the + // specified operation /* attributes used by host server to pass data to the server convenience library - the * data will then be parsed and provided to the local clients */ -#define PMIX_PROC_DATA "pmix.pdata" // (pmix_value_array_t) starts with rank, then contains more data -#define PMIX_NODE_MAP "pmix.nmap" // (char*) regex of nodes containing procs for this job -#define PMIX_PROC_MAP "pmix.pmap" // (char*) regex describing procs on each node within this job -#define PMIX_ANL_MAP "pmix.anlmap" // (char*) process mapping in ANL notation (used in PMI-1/PMI-2) +#define PMIX_PROC_DATA "pmix.pdata" // (pmix_value_array_t) starts with rank, then contains more data +#define PMIX_NODE_MAP "pmix.nmap" // (char*) regex of nodes containing procs for this job +#define PMIX_PROC_MAP "pmix.pmap" // (char*) regex describing procs on each node within this job +#define PMIX_ANL_MAP "pmix.anlmap" // (char*) process mapping in ANL notation (used in PMI-1/PMI-2) /* attributes used internally to communicate data from the server to the client */ -#define PMIX_PROC_BLOB "pmix.pblob" // (pmix_byte_object_t) packed blob of process data -#define PMIX_MAP_BLOB "pmix.mblob" // (pmix_byte_object_t) packed blob of process location +#define PMIX_PROC_BLOB "pmix.pblob" // (pmix_byte_object_t) packed blob of process data +#define PMIX_MAP_BLOB "pmix.mblob" // (pmix_byte_object_t) packed blob of process location -/* error handler registration and notification info keys */ -#define PMIX_ERROR_NAME "pmix.errname" // enum pmix_status_t specific error to be notified -#define PMIX_ERROR_GROUP_COMM "pmix.errgroup.comm" // bool - set true to get comm errors notification -#define PMIX_ERROR_GROUP_ABORT "pmix.errgroup.abort" // bool -set true to get abort errors notification -#define PMIX_ERROR_GROUP_MIGRATE "pmix.errgroup.migrate" // bool -set true to get migrate errors notification -#define PMIX_ERROR_GROUP_RESOURCE "pmix.errgroup.resource" // bool -set true to get resource errors notification -#define PMIX_ERROR_GROUP_SPAWN "pmix.errgroup.spawn" // bool - set true to get spawn errors notification -#define PMIX_ERROR_GROUP_NODE "pmix.errgroup.node" // bool -set true to get node status errors -#define PMIX_ERROR_GROUP_LOCAL "pmix.errgroup.local" // bool set true to get local errors -#define PMIX_ERROR_GROUP_GENERAL "pmix.errgroup.gen" // bool set true to get notified af generic errors -#define PMIX_ERROR_HANDLER_ID "pmix.errhandler.id" // int - errhandler reference id of notification being reported +/* event handler registration and notification info keys */ +#define PMIX_EVENT_HDLR_NAME "pmix.evname" // (char*) string name identifying this handler +#define PMIX_EVENT_JOB_LEVEL "pmix.evjob" // (bool) register for job-specific events only +#define PMIX_EVENT_ENVIRO_LEVEL "pmix.evenv" // (bool) register for environment events only +#define PMIX_EVENT_ORDER_PREPEND "pmix.evprepend" // (bool) prepend this handler to the precedence list +#define PMIX_EVENT_CUSTOM_RANGE "pmix.evrange" // (pmix_proc_t*) array of pmix_proc_t defining range of event notification +#define PMIX_EVENT_AFFECTED_PROCS "pmix.evaffected" // (pmix_proc_t*) array of pmix_proc_t defining affected procs +#define PMIX_EVENT_NON_DEFAULT "pmix.evnondef" // (bool) event is not to be delivered to default event handlers + /* fault tolerance-related events */ + #define PMIX_EVENT_TERMINATE_SESSION "pmix.evterm.sess" // (bool) RM intends to terminate session + #define PMIX_EVENT_TERMINATE_JOB "pmix.evterm.job" // (bool) RM intends to terminate this job + #define PMIX_EVENT_TERMINATE_NODE "pmix.evterm.node" // (bool) RM intends to terminate all procs on this node + #define PMIX_EVENT_TERMINATE_PROC "pmix.evterm.proc" // (bool) RM intends to terminate just this process + #define PMIX_EVENT_ACTION_TIMEOUT "pmix.evtimeout" // (int) time in sec before RM will execute error response -/* error notification keys */ -#define PMIX_ERROR_SCOPE "pmix.errscope" // int (enum pmix_scope_t) scope of error notification -#define PMIX_ERROR_NODE_NAME "pmix.errnode.name" // name of the node that is in error or which reported the error. -#define PMIX_ERROR_SEVERITY "pmix.errseverity" // the severity of the notified (reported) error - -/* attributes used to describe "spawm" attributes */ -#define PMIX_PERSONALITY "pmix.pers" // (char*) name of personality to use -#define PMIX_HOST "pmix.host" // (char*) comma-delimited list of hosts to use for spawned procs -#define PMIX_HOSTFILE "pmix.hostfile" // (char*) hostfile to use for spawned procs -#define PMIX_ADD_HOST "pmix.addhost" // (char*) comma-delimited list of hosts to add to allocation -#define PMIX_ADD_HOSTFILE "pmix.addhostfile" // (char*) hostfile to add to existing allocation -#define PMIX_PREFIX "pmix.prefix" // (char*) prefix to use for starting spawned procs -#define PMIX_WDIR "pmix.wdir" // (char*) working directory for spawned procs -#define PMIX_MAPPER "pmix.mapper" // (char*) mapper to use for placing spawned procs -#define PMIX_DISPLAY_MAP "pmix.dispmap" // (bool) display process map upon spawn -#define PMIX_PPR "pmix.ppr" // (char*) #procs to spawn on each identified resource -#define PMIX_MAPBY "pmix.mapby" // (char*) mapping policy -#define PMIX_RANKBY "pmix.rankby" // (char*) ranking policy -#define PMIX_BINDTO "pmix.bindto" // (char*) binding policy -#define PMIX_PRELOAD_BIN "pmix.preloadbin" // (bool) preload binaries -#define PMIX_PRELOAD_FILES "pmix.preloadfiles" // (char*) comma-delimited list of files to pre-position -#define PMIX_NON_PMI "pmix.nonpmi" // (bool) spawned procs will not call PMIx_Init -#define PMIX_STDIN_TGT "pmix.stdin" // (uint32_t) spawned proc rank that is to receive stdin +/* attributes used to describe "spawn" attributes */ +#define PMIX_PERSONALITY "pmix.pers" // (char*) name of personality to use +#define PMIX_HOST "pmix.host" // (char*) comma-delimited list of hosts to use for spawned procs +#define PMIX_HOSTFILE "pmix.hostfile" // (char*) hostfile to use for spawned procs +#define PMIX_ADD_HOST "pmix.addhost" // (char*) comma-delimited list of hosts to add to allocation +#define PMIX_ADD_HOSTFILE "pmix.addhostfile" // (char*) hostfile to add to existing allocation +#define PMIX_PREFIX "pmix.prefix" // (char*) prefix to use for starting spawned procs +#define PMIX_WDIR "pmix.wdir" // (char*) working directory for spawned procs +#define PMIX_MAPPER "pmix.mapper" // (char*) mapper to use for placing spawned procs +#define PMIX_DISPLAY_MAP "pmix.dispmap" // (bool) display process map upon spawn +#define PMIX_PPR "pmix.ppr" // (char*) #procs to spawn on each identified resource +#define PMIX_MAPBY "pmix.mapby" // (char*) mapping policy +#define PMIX_RANKBY "pmix.rankby" // (char*) ranking policy +#define PMIX_BINDTO "pmix.bindto" // (char*) binding policy +#define PMIX_PRELOAD_BIN "pmix.preloadbin" // (bool) preload binaries +#define PMIX_PRELOAD_FILES "pmix.preloadfiles" // (char*) comma-delimited list of files to pre-position +#define PMIX_NON_PMI "pmix.nonpmi" // (bool) spawned procs will not call PMIx_Init +#define PMIX_STDIN_TGT "pmix.stdin" // (uint32_t) spawned proc rank that is to receive stdin /**** PMIX ERROR CONSTANTS ****/ /* PMIx errors are always negative, with 0 reserved for success */ -#define PMIX_ERROR_MIN -50 // set equal to number of non-zero entries in enum +#define PMIX_ERR_BASE 0 -typedef enum { - PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER = PMIX_ERROR_MIN, - PMIX_ERR_COMM_FAILURE, - PMIX_ERR_NOT_IMPLEMENTED, - PMIX_ERR_NOT_SUPPORTED, - PMIX_ERR_NOT_FOUND, - PMIX_ERR_SERVER_NOT_AVAIL, - PMIX_ERR_INVALID_NAMESPACE, - PMIX_ERR_INVALID_SIZE, - PMIX_ERR_INVALID_KEYVALP, - PMIX_ERR_INVALID_NUM_PARSED, +typedef int pmix_status_t; - PMIX_ERR_INVALID_ARGS, - PMIX_ERR_INVALID_NUM_ARGS, - PMIX_ERR_INVALID_LENGTH, - PMIX_ERR_INVALID_VAL_LENGTH, - PMIX_ERR_INVALID_VAL, - PMIX_ERR_INVALID_KEY_LENGTH, - PMIX_ERR_INVALID_KEY, - PMIX_ERR_INVALID_ARG, - PMIX_ERR_NOMEM, - PMIX_ERR_INIT, +#define PMIX_SUCCESS (PMIX_ERR_BASE) +#define PMIX_ERROR (PMIX_ERR_BASE - 1) // general error +/* debugger release flag */ +#define PMIX_ERR_DEBUGGER_RELEASE (PMIX_ERR_BASE - 2) +/* fault tolerance */ +#define PMIX_ERR_PROC_RESTART (PMIX_ERR_BASE - 3) +#define PMIX_ERR_PROC_CHECKPOINT (PMIX_ERR_BASE - 4) +#define PMIX_ERR_PROC_MIGRATE (PMIX_ERR_BASE - 5) +#define PMIX_ERR_UPDATE_ENDPOINTS (PMIX_ERR_BASE - 6) +/* abort */ +#define PMIX_ERR_PROC_ABORTED (PMIX_ERR_BASE - 7) +#define PMIX_ERR_PROC_REQUESTED_ABORT (PMIX_ERR_BASE - 8) +#define PMIX_ERR_PROC_ABORTING (PMIX_ERR_BASE - 9) +/* communication failures */ +#define PMIX_ERR_UNREACH (PMIX_ERR_BASE - 10) +#define PMIX_ERR_LOST_CONNECTION_TO_SERVER (PMIX_ERR_BASE - 11) +#define PMIX_ERR_LOST_PEER_CONNECTION (PMIX_ERR_BASE - 12) +#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT (PMIX_ERR_BASE - 13) +/* operational */ +#define PMIX_ERR_NO_PERMISSIONS (PMIX_ERR_BASE - 14) +#define PMIX_ERR_TIMEOUT (PMIX_ERR_BASE - 15) +#define PMIX_ERR_WOULD_BLOCK (PMIX_ERR_BASE - 16) +#define PMIX_EXISTS (PMIX_ERR_BASE - 17) +#define PMIX_ERR_SERVER_FAILED_REQUEST (PMIX_ERR_BASE - 18) +#define PMIX_ERR_NOT_SUPPORTED (PMIX_ERR_BASE - 19) +#define PMIX_ERR_NOT_FOUND (PMIX_ERR_BASE - 20) +#define PMIX_ERR_BAD_PARAM (PMIX_ERR_BASE - 21) +#define PMIX_ERR_DATA_VALUE_NOT_FOUND (PMIX_ERR_BASE - 22) +#define PMIX_ERR_OUT_OF_RESOURCE (PMIX_ERR_BASE - 23) +#define PMIX_ERR_INVALID_NAMESPACE (PMIX_ERR_BASE - 24) +#define PMIX_ERR_INVALID_SIZE (PMIX_ERR_BASE - 25) +#define PMIX_ERR_INIT (PMIX_ERR_BASE - 26) +#define PMIX_ERR_EVENT_REGISTRATION (PMIX_ERR_BASE - 27) +/* system failures */ +#define PMIX_ERR_NODE_DOWN (PMIX_ERR_BASE - 28) +#define PMIX_ERR_NODE_OFFLINE (PMIX_ERR_BASE - 29) +/* used by event handlers */ +#define PMIX_EVENT_NO_ACTION_TAKEN (PMIX_ERR_BASE - 30) +#define PMIX_EVENT_PARTIAL_ACTION_TAKEN (PMIX_ERR_BASE - 31) +#define PMIX_EVENT_ACTION_DEFERRED (PMIX_ERR_BASE - 32) +#define PMIX_EVENT_ACTION_COMPLETE (PMIX_ERR_BASE - 33) - PMIX_ERR_DATA_VALUE_NOT_FOUND, - PMIX_ERR_OUT_OF_RESOURCE, - PMIX_ERR_RESOURCE_BUSY, - PMIX_ERR_BAD_PARAM, - PMIX_ERR_IN_ERRNO, - PMIX_ERR_UNREACH, - PMIX_ERR_TIMEOUT, - PMIX_ERR_NO_PERMISSIONS, - PMIX_ERR_PACK_MISMATCH, - PMIX_ERR_PACK_FAILURE, - PMIX_ERR_UNPACK_FAILURE, - PMIX_ERR_UNPACK_INADEQUATE_SPACE, - PMIX_ERR_TYPE_MISMATCH, - PMIX_ERR_PROC_ENTRY_NOT_FOUND, - PMIX_ERR_UNKNOWN_DATA_TYPE, - PMIX_ERR_WOULD_BLOCK, - PMIX_ERR_READY_FOR_HANDSHAKE, - PMIX_ERR_HANDSHAKE_FAILED, - PMIX_ERR_INVALID_CRED, - PMIX_EXISTS, +/* define a starting point for PMIx internal error codes + * that are never exposed outside the library */ +#define PMIX_INTERNAL_ERR_BASE -1000 - PMIX_ERR_SERVER_FAILED_REQUEST, - PMIX_ERR_PROC_ABORTING, - PMIX_ERR_PROC_REQUESTED_ABORT, - PMIX_ERR_PROC_ABORTED, - PMIX_ERR_PROC_MIGRATE, - PMIX_ERR_PROC_CHECKPOINT, - PMIX_ERR_PROC_RESTART, - PMIX_ERR_DEBUGGER_RELEASE, - PMIX_ERR_SILENT, - PMIX_ERROR, - - PMIX_SUCCESS -} pmix_status_t; +/* define a starting point for user-level defined error + * constants - negative values larger than this are guaranteed + * not to conflict with PMIx values. Definitions should always + * be based on the PMIX_EXTERNAL_ERR_BASE constant and -not- a + * specific value as the value of the constant may change */ +#define PMIX_EXTERNAL_ERR_BASE -2000 /**** PMIX DATA TYPES ****/ @@ -303,6 +306,9 @@ typedef enum { PMIX_TIMEVAL, PMIX_TIME, + PMIX_STATUS, // needs to be tracked separately from integer for those times + // when we are embedded and it needs to be converted to the + // host error definitions PMIX_HWLOC_TOPO, PMIX_VALUE, PMIX_INFO_ARRAY, @@ -314,7 +320,8 @@ typedef enum { PMIX_BYTE_OBJECT, PMIX_KVAL, PMIX_MODEX, - PMIX_PERSIST + PMIX_PERSIST, + PMIX_POINTER } pmix_data_type_t; /* define a scope for data "put" by PMI per the following: @@ -340,9 +347,13 @@ typedef enum { */ #define PMIX_DATA_RANGE PMIX_UINT typedef enum { - PMIX_DATA_RANGE_UNDEF = 0, - PMIX_NAMESPACE, // data is available to procs in the same nspace only - PMIX_SESSION // data available to all procs in session + PMIX_RANGE_UNDEF = 0, + PMIX_RANGE_RM, // data is intended for the host resource manager + PMIX_RANGE_LOCAL, // available on local node only + PMIX_RANGE_NAMESPACE, // data is available to procs in the same nspace only + PMIX_RANGE_SESSION, // data available to all procs in session + PMIX_RANGE_GLOBAL, // data available to all procs + PMIX_RANGE_CUSTOM // range is specified in a pmix_info_t } pmix_data_range_t; /* define a "persistence" policy for data published by clients */ @@ -426,8 +437,10 @@ typedef struct pmix_value { float fval; double dval; struct timeval tv; + pmix_status_t status; pmix_info_array_t array; pmix_byte_object_t bo; + void *ptr; } data; } pmix_value_t; /* allocate and initialize a specified number of value structs */ @@ -495,12 +508,13 @@ typedef struct pmix_value { } \ } while (0) -/* expose a function that is resolved in the +/* expose two functions that are resolved in the * PMIx library, but part of a header that - * includes internal functions - so we don't + * includes internal functions - we don't * want to expose the entire header here */ void pmix_value_load(pmix_value_t *v, void *data, pmix_data_type_t type); +pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); @@ -508,6 +522,7 @@ void pmix_value_load(pmix_value_t *v, void *data, pmix_data_type_t type); /**** PMIX INFO STRUCT ****/ struct pmix_info_t { char key[PMIX_MAX_KEYLEN+1]; // ensure room for the NULL terminator + bool required; // defaults to optional (i.e., required=false) pmix_value_t value; }; @@ -545,6 +560,17 @@ struct pmix_info_t { (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ pmix_value_load(&((m)->value), (v), (t)); \ } while (0) +#define PMIX_INFO_XFER(d, s) \ + do { \ + (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + (d)->required = (s)->required; \ + pmix_value_xfer(&(d)->value, &(s)->value); \ + } while(0) + +#define PMIX_INFO_REQUIRED(m) \ + (m)->required = true; +#define PMIX_INFO_OPTIONAL(m) \ + (m)->required = false; /**** PMIX LOOKUP RETURN STRUCT ****/ typedef struct pmix_pdata { @@ -754,31 +780,55 @@ typedef void (*pmix_lookup_cbfunc_t)(pmix_status_t status, pmix_pdata_t data[], size_t ndata, void *cbdata); -/* define a callback function for the errhandler. Upon receipt of an - * error notification, PMIx will execute the specified notification +/* define a callback by which an event handler can notify the PMIx library + * that it has completed its response to the notification. The handler + * is _required_ to execute this callback so the library can determine + * if additional handlers need to be called. The handler shall return + * PMIX_SUCCESS if no further action is required. The return status + * of each event handler and any returned pmix_info_t structures + * will be added to the array of pmix_info_t passed to any subsequent + * event handlers to help guide their operation. + * + * If non-NULL, the provided callback function will be called to allow + * the event handler to release the provided info array. + */ +typedef void (*pmix_event_notification_cbfunc_fn_t)(pmix_status_t status, + pmix_info_t *results, size_t nresults, + pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata); + +/* define a callback function for the event handler. Upon receipt of an + * event notification, PMIx will execute the specified notification * callback function, providing: * - * status - the error that occurred - * procs - the nspace and ranks of the affected processes. A NULL - * value indicates that the error occurred in the PMIx - * client library within this process itself - * nprocs - the number of procs in the provided array - * info - any additional info provided regarding the error. - * ninfo - the number of info objects in the provided array + * evhdlr_registration_id - the returned registration number of + * the event handler being called + * status - the event that occurred + * source - the nspace and rank of the process that generated + * the event. If the source is the resource manager, + * then the nspace will be empty and the rank will + * be PMIX_RANK_UNDEF + * info - any additional info provided regarding the event. + * ninfo - the number of info objects in the info array + * results - any provided results from event handlers called + * prior to this one. + * nresults - number of info objects in the results array + * cbfunc - the function to be called upon completion of the handler + * cbdata - pointer to be returned in the completion cbfunc * * Note that different resource managers may provide differing levels - * of support for error notification to application processes. Thus, the - * info array may be NULL or may contain detailed information of the error. + * of support for event notification to application processes. Thus, the + * info array may be NULL or may contain detailed information of the event. * It is the responsibility of the application to parse any provided info array * for defined key-values if it so desires. * * Possible uses of the pmix_info_t object include: * * - for the RM to alert the process as to planned actions, such as - * to abort the session, in response to the reported error + * to abort the session, in response to the reported event * * - provide a timeout for alternative action to occur, such as for - * the application to request an alternate response to the error + * the application to request an alternate response to the event * * For example, the RM might alert the application to the failure of * a node that resulted in termination of several processes, and indicate @@ -793,23 +843,24 @@ typedef void (*pmix_lookup_cbfunc_t)(pmix_status_t status, * on a per-RM basis. * * On the server side, the notification function is used to inform the host - * server of a detected error in the PMIx subsystem and/or client - * - * The errhandler_ref is included as the first pmix_info_t in the returned - * array for embedded scenarios where the notification callback is to a switchyard + * server of a detected event in the PMIx subsystem and/or client */ -typedef void (*pmix_notification_fn_t)(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo); +typedef void (*pmix_notification_fn_t)(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t *results, size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); -/* define a callback function for calls to PMIx_Register_errhandler. The - * status indicates if the request was successful or not, errhandler_ref is - * an integer reference assigned to the errhandler by PMIX, this reference +/* define a callback function for calls to PMIx_Register_evhdlr. The + * status indicates if the request was successful or not, evhdlr_ref is + * an integer reference assigned to the event handler by PMIx, this reference * must be used to deregister the err handler. A ptr to the original * cbdata is returned. */ -typedef void (*pmix_errhandler_reg_cbfunc_t)(pmix_status_t status, - int errhandler_ref, - void *cbdata); +typedef void (*pmix_evhdlr_reg_cbfunc_t)(pmix_status_t status, + size_t evhdlr_ref, + void *cbdata); /* define a callback function for calls to PMIx_Get_nb. The status * indicates if the requested data was found or not - a pointer to the @@ -819,89 +870,86 @@ typedef void (*pmix_value_cbfunc_t)(pmix_status_t status, pmix_value_t *kv, void *cbdata); /**** COMMON SUPPORT FUNCTIONS ****/ -/* Register an errhandler to report errors. Three types of errors +/* Register an event handler to report events. Three types of events * can be reported: * * (a) those that occur within the client library, but are not * reportable via the API itself (e.g., loss of connection to - * the server). These errors typically occur during behind-the-scenes + * the server). These events typically occur during behind-the-scenes * non-blocking operations. * - * (b) job-related errors such as the failure of another process in + * (b) job-related events such as the failure of another process in * the job or in any connected job, impending failure of hardware * within the job's usage footprint, etc. * * (c) system notifications that are made available by the local * administrators * - * By default, only errors that directly affect the process and/or + * By default, only events that directly affect the process and/or * any process to which it is connected (via the PMIx_Connect call) * will be reported. Options to modify that behavior can be provided * in the info array * * Both the client application and the resource manager can register - * err handlers for specific errors. PMIx client/server calls the registered - * err handler upon receiving error notify notification (via PMIx_Notify_error) + * err handlers for specific events. PMIx client/server calls the registered + * err handler upon receiving event notify notification (via PMIx_Notify_event) * from the other end (Resource Manager/Client application). * - * Multiple err handlers can be registered for different errors. PMIX returns + * Multiple err handlers can be registered for different events. PMIX returns * an integer reference to each register handler in the callback fn. The caller - * must retain the reference in order to deregister the errhandler. + * must retain the reference in order to deregister the evhdlr. * Modification of the notification behavior can be accomplished by - * deregistering the current errhandler, and then registering it + * deregistering the current evhdlr, and then registering it * using a new set of info values. * * See pmix_common.h for a description of the notification function */ -void PMIx_Register_errhandler(pmix_info_t info[], size_t ninfo, - pmix_notification_fn_t errhandler, - pmix_errhandler_reg_cbfunc_t cbfunc, - void *cbdata); +void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes, + pmix_info_t info[], size_t ninfo, + pmix_notification_fn_t evhdlr, + pmix_evhdlr_reg_cbfunc_t cbfunc, + void *cbdata); -/* deregister the errhandler - * errhandler_ref is the reference returned by PMIx for the errhandler - * to pmix_errhandler_reg_cbfunc_t */ -void PMIx_Deregister_errhandler(int errhandler_ref, - pmix_op_cbfunc_t cbfunc, - void *cbdata); -/* Report an error to a process for notification via any - * registered errhandler. The errhandler registration can be +/* Deregister an event handler + * evhdlr_ref is the reference returned by PMIx from the call to + * PMIx_Register_event_handler. If non-NULL, the provided cbfunc + * will be called to confirm removal of the designated handler */ +void PMIx_Deregister_event_handler(size_t evhdlr_ref, + pmix_op_cbfunc_t cbfunc, + void *cbdata); + +/* Report an event to a process for notification via any + * registered evhdlr. The evhdlr registration can be * called by both the server and the client application. On the - * server side, the errhandler is used to report errors detected + * server side, the evhdlr is used to report events detected * by PMIx to the host server for handling. On the client side, - * the errhandler is used to notify the process of errors + * the evhdlr is used to notify the process of events * reported by the server - e.g., the failure of another process. * * This function allows the host server to direct the server - * convenience library to notify all indicated local procs of - * an error. The error can be local, or anywhere in the cluster. - * The status indicates the error being reported. + * convenience library to notify all registered local procs of + * an event. The event can be local, or anywhere in the cluster. + * The status indicates the event being reported. * * The client application can also call this function to notify the - * resource manager of an error it encountered. It can request the host - * server to notify the indicated processes about the error. + * resource manager of an event it encountered. It can request the host + * server to notify the indicated processes about the event. * - * The first array of procs informs the server library as to which - * processes should be alerted - e.g., the processes that are in - * a directly-affected job or are connected to one that is affected. - * Passing a NULL for this array will indicate that all local procs - * are to be notified. - * - * The second array identifies the processes that will be impacted - * by the error. This could consist of a single process, or a number + * The array of procs identifies the processes that will be impacted + * by the event. This could consist of a single process, or a number * of processes. * * The info array contains any further info the RM can and/or chooses * to provide. * * The callback function will be called upon completion of the - * notify_error function's actions. Note that any messages will + * notify_event function's actions. Note that any messages will * have been queued, but may not have been transmitted by this * time. Note that the caller is required to maintain the input * data until the callback function has been executed! */ -pmix_status_t PMIx_Notify_error(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_proc_t error_procs[], size_t error_nprocs, +pmix_status_t PMIx_Notify_event(pmix_status_t status, + const pmix_proc_t *source, + pmix_data_range_t range, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix/rename.h b/opal/mca/pmix/pmix2x/pmix/include/pmix/rename.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/include/pmix/rename.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix/rename.h diff --git a/opal/mca/pmix/pmix114/pmix/include/pmix_server.h b/opal/mca/pmix/pmix2x/pmix/include/pmix_server.h similarity index 85% rename from opal/mca/pmix/pmix114/pmix/include/pmix_server.h rename to opal/mca/pmix/pmix2x/pmix/include/pmix_server.h index b2fdacd929..90b286da7d 100644 --- a/opal/mca/pmix/pmix114/pmix/include/pmix_server.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_server.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -82,9 +82,12 @@ BEGIN_C_DECLS * server, which is free to release it upon return from the callback */ -/* Notify the host server that a client connected to us */ -typedef pmix_status_t (*pmix_server_client_connected_fn_t)(const pmix_proc_t *proc, - void* server_object); +/* Notify the host server that a client connected to us - note + * that the client will be in a blocked state until the host server + * executes the callback function, thus allowing the PMIx server support + * library to release the client */ +typedef pmix_status_t (*pmix_server_client_connected_fn_t)(const pmix_proc_t *proc, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Notify the host server that a client called PMIx_Finalize - note * that the client will be in a blocked state until the host server @@ -235,25 +238,41 @@ typedef pmix_status_t (*pmix_server_disconnect_fn_t)(const pmix_proc_t procs[], pmix_op_cbfunc_t cbfunc, void *cbdata); /* Register to receive notifications for the specified events. The resource - * manager may have access to events beyond process failure. The client - * application requests to be notified of such events by registering a - * err handler(s) for the events. The PMIx client shall pass the request - * to the PMIx server, which in turn shall pass the request to - * the resource manager by calling the register events function. */ - typedef pmix_status_t (*pmix_server_register_events_fn_t)(const pmix_info_t info[], size_t ninfo, + * manager is _required_ to pass along to the local PMIx server all events + * that directly relate to a registered namespace. However, the RM may have + * access to events beyond those - e.g., environmental events. The PMIx server + * will register to receive environmental events that match specific PMIx + * event codes. If the host RM supports such notifications, it will need to + * translate its own internal event codes to fit into a corresponding PMIx event + * code - any specific info beyond that can be passed in via the pmix_info_t + * upon notification. + * + * The info array included in this API is reserved for possible future directives + * to further steer notification. + */ + typedef pmix_status_t (*pmix_server_register_events_fn_t)(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); -/* Deregister to receive notifications for the specified events that - * the client application has registered for previously. When the client - * application deregisters the err handler forevents, PMIX client passes the - * deregister request to PMIx server which in turn passes the request to the - * resource manager by calling deregister events function.*/ - typedef pmix_status_t (*pmix_server_deregister_events_fn_t)(const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); +/* Deregister to receive notifications for the specified environmental events + * for which the PMIx server has previously registered. The host RM remains + * required to notify of any job-related events */ + typedef pmix_status_t (*pmix_server_deregister_events_fn_t)(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/* Notify the specified processes of an event generated either by + * the PMIx server itself, or by one of its local clients. The process + * generating the event is provided in the source parameter. */ +typedef pmix_status_t (*pmix_server_notify_event_fn_t)(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + /* Callback function for incoming connection requests from * local clients */ -typedef void (*pmix_connection_cbfunc_t)(int incoming_sd); +typedef void (*pmix_connection_cbfunc_t)(int incoming_sd, void *cbdata); /* Register a socket the host server can monitor for connection * requests, harvest them, and then call our internal callback @@ -266,23 +285,25 @@ typedef void (*pmix_connection_cbfunc_t)(int incoming_sd); * will cause the internal PMIx server to spawn its own listener * thread */ typedef pmix_status_t (*pmix_server_listener_fn_t)(int listening_sd, - pmix_connection_cbfunc_t cbfunc); + pmix_connection_cbfunc_t cbfunc, + void *cbdata); -typedef struct pmix_server_module_1_0_0_t { - pmix_server_client_connected_fn_t client_connected; - pmix_server_client_finalized_fn_t client_finalized; - pmix_server_abort_fn_t abort; - pmix_server_fencenb_fn_t fence_nb; - pmix_server_dmodex_req_fn_t direct_modex; - pmix_server_publish_fn_t publish; - pmix_server_lookup_fn_t lookup; - pmix_server_unpublish_fn_t unpublish; - pmix_server_spawn_fn_t spawn; - pmix_server_connect_fn_t connect; - pmix_server_disconnect_fn_t disconnect; - pmix_server_register_events_fn_t register_events; - pmix_server_deregister_events_fn_t deregister_events; - pmix_server_listener_fn_t listener; +typedef struct pmix_server_module_2_0_0_t { + pmix_server_client_connected_fn_t client_connected; + pmix_server_client_finalized_fn_t client_finalized; + pmix_server_abort_fn_t abort; + pmix_server_fencenb_fn_t fence_nb; + pmix_server_dmodex_req_fn_t direct_modex; + pmix_server_publish_fn_t publish; + pmix_server_lookup_fn_t lookup; + pmix_server_unpublish_fn_t unpublish; + pmix_server_spawn_fn_t spawn; + pmix_server_connect_fn_t connect; + pmix_server_disconnect_fn_t disconnect; + pmix_server_register_events_fn_t register_events; + pmix_server_deregister_events_fn_t deregister_events; + pmix_server_notify_event_fn_t notify_event; + pmix_server_listener_fn_t listener; } pmix_server_module_t; /**** SERVER SUPPORT INIT/FINALIZE FUNCTIONS ****/ @@ -367,7 +388,8 @@ pmix_status_t PMIx_server_register_nspace(const char nspace[], int nlocalprocs, * intended to support persistent PMIx servers by providing * an opportunity for the host RM to tell the PMIx server * library to release all memory for a completed job */ -void PMIx_server_deregister_nspace(const char nspace[]); +void PMIx_server_deregister_nspace(const char nspace[], + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Register a client process with the PMIx server library. The * expected user ID and group ID of the child process helps the @@ -391,7 +413,8 @@ pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, * deregister_nspace API will automatically delete all client * info for that nspace - this API is therefore intended solely * for use in exception cases */ -void PMIx_server_deregister_client(const pmix_proc_t *proc); +void PMIx_server_deregister_client(const pmix_proc_t *proc, + pmix_op_cbfunc_t cbfunc, void *cbdata); /* Setup the environment of a child process to be forked * by the host so it can correctly interact with the PMIx diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_abort.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_abort.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_abort.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_abort.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_commit.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_commit.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_commit.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_commit.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_fence.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_fence.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_fence.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_fence.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_finalize.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_finalize.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_finalize.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_finalize.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_get.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_get.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_get.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_get.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_init.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_init.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_initialized.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_initialized.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_initialized.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_initialized.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man3/pmix_put.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man3/pmix_put.3 rename to opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 diff --git a/opal/mca/pmix/pmix114/pmix/man/man7/pmix.7 b/opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man7/pmix.7 rename to opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 diff --git a/opal/mca/pmix/pmix114/pmix/man/man7/pmix_constants.7 b/opal/mca/pmix/pmix2x/pmix/man/man7/pmix_constants.7 similarity index 100% rename from opal/mca/pmix/pmix114/pmix/man/man7/pmix_constants.7 rename to opal/mca/pmix/pmix2x/pmix/man/man7/pmix_constants.7 diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/buffer_ops.h b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/buffer_ops.h similarity index 95% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/buffer_ops.h rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/buffer_ops.h index fdb591102d..dca688e538 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/buffer_ops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/buffer_ops.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -44,12 +44,12 @@ BEGIN_C_DECLS * of places throughout the code base - transferring a value to * another pmix_value_t structure */ -PMIX_DECLSPEC pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); -PMIX_DECLSPEC void pmix_value_load(pmix_value_t *v, void *data, +pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); +void pmix_value_load(pmix_value_t *v, void *data, pmix_data_type_t type); -PMIX_DECLSPEC pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, +pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, size_t *sz, pmix_data_type_t type); -PMIX_DECLSPEC bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1); +bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1); #define PMIX_LOAD_BUFFER(b, d, s) \ @@ -247,12 +247,12 @@ typedef pmix_status_t (*pmix_bfrop_copy_payload_fn_t)(pmix_buffer_t *dest, * structure gets loaded, so we provide an "open" call that is * executed as part of the program startup. */ -PMIX_DECLSPEC pmix_status_t pmix_bfrop_open(void); +pmix_status_t pmix_bfrop_open(void); /** * BFROP finalize function */ -PMIX_DECLSPEC pmix_status_t pmix_bfrop_close(void); +pmix_status_t pmix_bfrop_close(void); /** @@ -308,7 +308,7 @@ struct pmix_bfrop_t { }; typedef struct pmix_bfrop_t pmix_bfrop_t; -PMIX_DECLSPEC extern pmix_bfrop_t pmix_bfrop; /* holds bfrop function pointers */ +extern pmix_bfrop_t pmix_bfrop; /* holds bfrop function pointers */ END_C_DECLS diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/copy.c b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/copy.c similarity index 96% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/copy.c rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/copy.c index ba837ac0ae..5f2669d67d 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/copy.c +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/copy.c @@ -137,6 +137,14 @@ pmix_status_t pmix_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src) datasize = sizeof(time_t); break; + case PMIX_STATUS: + datasize = sizeof(pmix_status_t); + break; + + case PMIX_POINTER: + datasize = sizeof(char*); + break; + default: return PMIX_ERR_UNKNOWN_DATA_TYPE; } @@ -167,7 +175,7 @@ pmix_status_t pmix_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src) return PMIX_SUCCESS; } -/* compare function for pmix_value_t*/ +/* compare function for pmix_value_t */ bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1) { bool rc = false; @@ -214,6 +222,9 @@ bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1) case PMIX_STRING: rc = strcmp(p->data.string, p1->data.string); break; + case PMIX_STATUS: + rc = (p->data.status == p1->data.status); + break; default: pmix_output(0, "COMPARE-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)p->type); } @@ -294,6 +305,9 @@ pmix_status_t pmix_value_xfer(pmix_value_t *p, pmix_value_t *src) p->data.tv.tv_sec = src->data.tv.tv_sec; p->data.tv.tv_usec = src->data.tv.tv_usec; break; + case PMIX_STATUS: + memcpy(&p->data.status, &src->data.status, sizeof(pmix_status_t)); + break; case PMIX_INFO_ARRAY: p->data.array.size = src->data.array.size; if (0 < src->data.array.size) { @@ -313,6 +327,9 @@ pmix_status_t pmix_value_xfer(pmix_value_t *p, pmix_value_t *src) p->data.bo.size = 0; } break; + case PMIX_POINTER: + memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); + break; default: pmix_output(0, "COPY-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); return PMIX_ERROR; @@ -344,6 +361,7 @@ pmix_status_t pmix_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + (*dest)->required = src->required; return pmix_value_xfer(&(*dest)->value, &src->value); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/internal.h b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/internal.h new file mode 100644 index 0000000000..5d3fd3ea33 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/internal.h @@ -0,0 +1,484 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#ifndef PMIX_BFROP_INTERNAL_H_ +#define PMIX_BFROP_INTERNAL_H_ + +#include + + +#ifdef HAVE_SYS_TIME_H +#include /* for struct timeval */ +#endif + +#include "src/class/pmix_pointer_array.h" + +#include "buffer_ops.h" + +#ifdef HAVE_STRING_H +#include +#endif +#if PMIX_HAVE_HWLOC +#include PMIX_HWLOC_HEADER +#endif + + BEGIN_C_DECLS + +/* + * The default starting chunk size + */ +#define PMIX_BFROP_DEFAULT_INITIAL_SIZE 128 +/* + * The default threshold size when we switch from doubling the + * buffer size to addatively increasing it + */ +#define PMIX_BFROP_DEFAULT_THRESHOLD_SIZE 1024 + +/* + * Internal type corresponding to size_t. Do not use this in + * interface calls - use PMIX_SIZE instead. + */ +#if SIZEOF_SIZE_T == 1 +#define BFROP_TYPE_SIZE_T PMIX_UINT8 +#elif SIZEOF_SIZE_T == 2 +#define BFROP_TYPE_SIZE_T PMIX_UINT16 +#elif SIZEOF_SIZE_T == 4 +#define BFROP_TYPE_SIZE_T PMIX_UINT32 +#elif SIZEOF_SIZE_T == 8 +#define BFROP_TYPE_SIZE_T PMIX_UINT64 +#else +#error Unsupported size_t size! +#endif + +/* + * Internal type corresponding to bool. Do not use this in interface + * calls - use PMIX_BOOL instead. + */ +#if SIZEOF__BOOL == 1 +#define BFROP_TYPE_BOOL PMIX_UINT8 +#elif SIZEOF__BOOL == 2 +#define BFROP_TYPE_BOOL PMIX_UINT16 +#elif SIZEOF__BOOL == 4 +#define BFROP_TYPE_BOOL PMIX_UINT32 +#elif SIZEOF__BOOL == 8 +#define BFROP_TYPE_BOOL PMIX_UINT64 +#else +#error Unsupported bool size! +#endif + +/* + * Internal type corresponding to int and unsigned int. Do not use + * this in interface calls - use PMIX_INT / PMIX_UINT instead. + */ +#if SIZEOF_INT == 1 +#define BFROP_TYPE_INT PMIX_INT8 +#define BFROP_TYPE_UINT PMIX_UINT8 +#elif SIZEOF_INT == 2 +#define BFROP_TYPE_INT PMIX_INT16 +#define BFROP_TYPE_UINT PMIX_UINT16 +#elif SIZEOF_INT == 4 +#define BFROP_TYPE_INT PMIX_INT32 +#define BFROP_TYPE_UINT PMIX_UINT32 +#elif SIZEOF_INT == 8 +#define BFROP_TYPE_INT PMIX_INT64 +#define BFROP_TYPE_UINT PMIX_UINT64 +#else +#error Unsupported int size! +#endif + +/* + * Internal type corresponding to pid_t. Do not use this in interface + * calls - use PMIX_PID instead. + */ +#if SIZEOF_PID_T == 1 +#define BFROP_TYPE_PID_T PMIX_UINT8 +#elif SIZEOF_PID_T == 2 +#define BFROP_TYPE_PID_T PMIX_UINT16 +#elif SIZEOF_PID_T == 4 +#define BFROP_TYPE_PID_T PMIX_UINT32 +#elif SIZEOF_PID_T == 8 +#define BFROP_TYPE_PID_T PMIX_UINT64 +#else +#error Unsupported pid_t size! +#endif + +/* Unpack generic size macros */ +#define UNPACK_SIZE_MISMATCH(unpack_type, remote_type, ret) \ + do { \ + switch(remote_type) { \ + case PMIX_UINT8: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint8_t, remote_type); \ + break; \ + case PMIX_INT8: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int8_t, remote_type); \ + break; \ + case PMIX_UINT16: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint16_t, remote_type); \ + break; \ + case PMIX_INT16: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int16_t, remote_type); \ + break; \ + case PMIX_UINT32: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint32_t, remote_type); \ + break; \ + case PMIX_INT32: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int32_t, remote_type); \ + break; \ + case PMIX_UINT64: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, uint64_t, remote_type); \ + break; \ + case PMIX_INT64: \ + UNPACK_SIZE_MISMATCH_FOUND(unpack_type, int64_t, remote_type); \ + break; \ + default: \ + ret = PMIX_ERR_NOT_FOUND; \ + } \ +} while (0) + +/* NOTE: do not need to deal with endianness here, as the unpacking of + the underling sender-side type will do that for us. Repeat: the + data in tmpbuf[] is already in host byte order. */ +#define UNPACK_SIZE_MISMATCH_FOUND(unpack_type, tmptype, tmpbfroptype) \ + do { \ + int32_t i; \ + tmptype *tmpbuf = (tmptype*)malloc(sizeof(tmptype) * (*num_vals)); \ + ret = pmix_bfrop_unpack_buffer(buffer, tmpbuf, num_vals, tmpbfroptype); \ + for (i = 0 ; i < *num_vals ; ++i) { \ + ((unpack_type*) dest)[i] = (unpack_type)(tmpbuf[i]); \ + } \ + free(tmpbuf); \ +} while (0) + + +/** + * Internal struct used for holding registered bfrop functions + */ + typedef struct { + pmix_object_t super; + /* type identifier */ + pmix_data_type_t odti_type; + /** Debugging string name */ + char *odti_name; + /** Pack function */ + pmix_bfrop_pack_fn_t odti_pack_fn; + /** Unpack function */ + pmix_bfrop_unpack_fn_t odti_unpack_fn; + /** copy function */ + pmix_bfrop_copy_fn_t odti_copy_fn; + /** print function */ + pmix_bfrop_print_fn_t odti_print_fn; +} pmix_bfrop_type_info_t; +PMIX_CLASS_DECLARATION(pmix_bfrop_type_info_t); + +/* + * globals needed within bfrop + */ + extern bool pmix_bfrop_initialized; + extern size_t pmix_bfrop_initial_size; + extern size_t pmix_bfrop_threshold_size; + extern pmix_pointer_array_t pmix_bfrop_types; + extern pmix_data_type_t pmix_bfrop_num_reg_types; + +/* macro for registering data types */ +#define PMIX_REGISTER_TYPE(n, t, p, u, c, pr) \ + do { \ + pmix_bfrop_type_info_t *_info; \ + _info = PMIX_NEW(pmix_bfrop_type_info_t); \ + _info->odti_name = strdup((n)); \ + _info->odti_type = (t); \ + _info->odti_pack_fn = (pmix_bfrop_pack_fn_t)(p); \ + _info->odti_unpack_fn = (pmix_bfrop_unpack_fn_t)(u); \ + _info->odti_copy_fn = (pmix_bfrop_copy_fn_t)(c) ; \ + _info->odti_print_fn = (pmix_bfrop_print_fn_t)(pr) ; \ + pmix_pointer_array_set_item(&pmix_bfrop_types, (t), _info); \ + ++pmix_bfrop_num_reg_types; \ +} while (0) + +/* + * Implementations of API functions + */ + + pmix_status_t pmix_bfrop_pack(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dest, + int32_t *max_num_vals, + pmix_data_type_t type); + + pmix_status_t pmix_bfrop_copy(void **dest, void *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_print(char **output, char *prefix, void *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src); + +/* + * Specialized functions + */ + pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_unpack_buffer(pmix_buffer_t *buffer, void *dst, + int32_t *num_vals, pmix_data_type_t type); + +/* + * Internal pack functions + */ + + pmix_status_t pmix_bfrop_pack_bool(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_byte(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_string(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_sizet(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_pid(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_pack_int(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_int16(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_int32(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_datatype(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_int64(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_pack_float(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_double(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +pmix_status_t pmix_bfrop_pack_time(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_timeval(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_time(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +pmix_status_t pmix_bfrop_pack_status(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + +#if PMIX_HAVE_HWLOC + pmix_status_t pmix_bfrop_pack_topo(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +#endif + pmix_status_t pmix_bfrop_pack_value(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_proc(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_app(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_info(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_buf(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_kval(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_persist(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_bo(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_pdata(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_pack_ptr(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); + +/* + * Internal unpack functions + */ + pmix_status_t pmix_bfrop_unpack_bool(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_byte(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_string(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_sizet(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_pid(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_unpack_int(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_int16(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_int32(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_datatype(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_int64(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_unpack_float(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_double(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_timeval(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_time(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +pmix_status_t pmix_bfrop_unpack_status(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + +#if PMIX_HAVE_HWLOC + pmix_status_t pmix_bfrop_unpack_topo(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +#endif + pmix_status_t pmix_bfrop_unpack_value(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_proc(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_app(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_buf(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_kval(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_persist(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_bo(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + pmix_status_t pmix_bfrop_unpack_ptr(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); + +/* + * Internal copy functions + */ + + pmix_status_t pmix_bfrop_std_copy(void **dest, void *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_copy_string(char **dest, char *src, pmix_data_type_t type); + +#if PMIX_HAVE_HWLOC + pmix_status_t pmix_bfrop_copy_topo(hwloc_topology_t *dest, + hwloc_topology_t src, + pmix_data_type_t type); +#endif + pmix_status_t pmix_bfrop_copy_value(pmix_value_t **dest, pmix_value_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_array(pmix_info_array_t **dest, pmix_info_array_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_buf(pmix_buffer_t **dest, pmix_buffer_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_kval(pmix_kval_t **dest, pmix_kval_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_bo(pmix_byte_object_t **dest, pmix_byte_object_t *src, + pmix_data_type_t type); + pmix_status_t pmix_bfrop_copy_pdata(pmix_pdata_t **dest, pmix_pdata_t *src, + pmix_data_type_t type); +/* + * Internal print functions + */ + pmix_status_t pmix_bfrop_print_bool(char **output, char *prefix, bool *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_byte(char **output, char *prefix, uint8_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_string(char **output, char *prefix, char *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_size(char **output, char *prefix, size_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_pid(char **output, char *prefix, pid_t *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_print_int(char **output, char *prefix, int *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_int8(char **output, char *prefix, int8_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_int16(char **output, char *prefix, int16_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_int32(char **output, char *prefix, int32_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_int64(char **output, char *prefix, int64_t *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_print_uint(char **output, char *prefix, uint *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_uint8(char **output, char *prefix, uint8_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_uint16(char **output, char *prefix, uint16_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_uint32(char **output, char *prefix, uint32_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_uint64(char **output, char *prefix, uint64_t *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_print_float(char **output, char *prefix, float *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_double(char **output, char *prefix, double *src, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_print_timeval(char **output, char *prefix, struct timeval *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_time(char **output, char *prefix, time_t *src, pmix_data_type_t type); +pmix_status_t pmix_bfrop_print_status(char **output, char *prefix, pmix_status_t *src, pmix_data_type_t type); + +#if PMIX_HAVE_HWLOC + pmix_status_t pmix_bfrop_print_topo(char **output, char *prefix, + hwloc_topology_t src, pmix_data_type_t type); +#endif + pmix_status_t pmix_bfrop_print_value(char **output, char *prefix, pmix_value_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_proc(char **output, char *prefix, + pmix_proc_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_app(char **output, char *prefix, + pmix_app_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_info(char **output, char *prefix, + pmix_info_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_buf(char **output, char *prefix, + pmix_buffer_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_kval(char **output, char *prefix, + pmix_kval_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_persist(char **output, char *prefix, + pmix_persistence_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_bo(char **output, char *prefix, + pmix_byte_object_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_pdata(char **output, char *prefix, + pmix_pdata_t *src, pmix_data_type_t type); + pmix_status_t pmix_bfrop_print_ptr(char **output, char *prefix, + void *src, pmix_data_type_t type); + +/* + * Internal helper functions + */ + + char* pmix_bfrop_buffer_extend(pmix_buffer_t *bptr, size_t bytes_to_add); + + bool pmix_bfrop_too_small(pmix_buffer_t *buffer, size_t bytes_reqd); + + pmix_bfrop_type_info_t* pmix_bfrop_find_type(pmix_data_type_t type); + + pmix_status_t pmix_bfrop_store_data_type(pmix_buffer_t *buffer, pmix_data_type_t type); + + pmix_status_t pmix_bfrop_get_data_type(pmix_buffer_t *buffer, pmix_data_type_t *type); + + END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/internal_functions.c b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/internal_functions.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/internal_functions.c rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/internal_functions.c diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/open_close.c b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/open_close.c similarity index 94% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/open_close.c rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/open_close.c index 2c8f67f05b..52a487acf8 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/open_close.c +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/open_close.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -33,6 +33,7 @@ #endif #include "src/util/argv.h" +#include "src/util/error.h" #include "src/buffer_ops/internal.h" /** @@ -45,10 +46,7 @@ pmix_pointer_array_t pmix_bfrop_types = {{0}}; pmix_data_type_t pmix_bfrop_num_reg_types = PMIX_UNDEF; static pmix_bfrop_buffer_type_t pmix_default_buf_type = PMIX_BFROP_BUFFER_NON_DESC; -PMIX_EXPORT pmix_status_t pmix_bfrop_open(void); -PMIX_EXPORT pmix_status_t pmix_bfrop_close(void); - -PMIX_EXPORT pmix_bfrop_t pmix_bfrop = { +pmix_bfrop_t pmix_bfrop = { pmix_bfrop_pack, pmix_bfrop_unpack, pmix_bfrop_copy, @@ -293,6 +291,12 @@ pmix_status_t pmix_bfrop_open(void) pmix_bfrop_std_copy, pmix_bfrop_print_time); + PMIX_REGISTER_TYPE("PMIX_STATUS", PMIX_STATUS, + pmix_bfrop_pack_status, + pmix_bfrop_unpack_status, + pmix_bfrop_std_copy, + pmix_bfrop_print_status); + #if PMIX_HAVE_HWLOC PMIX_REGISTER_TYPE("PMIX_HWLOC_TOPO", PMIX_HWLOC_TOPO, pmix_bfrop_pack_topo, @@ -367,6 +371,12 @@ pmix_status_t pmix_bfrop_open(void) pmix_bfrop_copy_persist, pmix_bfrop_print_persist); + PMIX_REGISTER_TYPE("PMIX_POINTER", PMIX_POINTER, + pmix_bfrop_pack_ptr, + pmix_bfrop_unpack_ptr, + pmix_bfrop_std_copy, + pmix_bfrop_print_ptr); + /* All done */ pmix_bfrop_initialized = true; return PMIX_SUCCESS; @@ -463,11 +473,17 @@ PMIX_EXPORT void pmix_value_load(pmix_value_t *v, void *data, case PMIX_TIMEVAL: memcpy(&(v->data.tv), data, sizeof(struct timeval)); break; + case PMIX_STATUS: + memcpy(&(v->data.status), data, sizeof(pmix_status_t)); + break; case PMIX_BYTE_OBJECT: bo = (pmix_byte_object_t*)data; v->data.bo.bytes = bo->bytes; memcpy(&(v->data.bo.size), &bo->size, sizeof(size_t)); break; + case PMIX_POINTER: + memcpy(&(v->data.ptr), data, sizeof(void*)); + break; case PMIX_TIME: case PMIX_HWLOC_TOPO: case PMIX_VALUE: @@ -576,6 +592,10 @@ pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, memcpy(*data, &(kv->data.tv), sizeof(struct timeval)); *sz = sizeof(struct timeval); break; + case PMIX_STATUS: + memcpy(*data, &(kv->data.status), sizeof(pmix_status_t)); + *sz = sizeof(pmix_status_t); + break; case PMIX_BYTE_OBJECT: if (NULL != kv->data.bo.bytes && 0 < kv->data.bo.size) { *data = kv->data.bo.bytes; @@ -585,6 +605,10 @@ pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, *sz = 0; } break; + case PMIX_POINTER: + memcpy(*data, &(kv->data.ptr), sizeof(void*)); + *sz = sizeof(void*); + break; case PMIX_TIME: case PMIX_HWLOC_TOPO: case PMIX_VALUE: diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/pack.c b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/pack.c similarity index 87% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/pack.c rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/pack.c index fe900d8ba0..a8db4a502a 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/pack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/pack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -37,9 +37,9 @@ #include "src/buffer_ops/internal.h" pmix_status_t pmix_bfrop_pack(pmix_buffer_t *buffer, - const void *src, int32_t num_vals, - pmix_data_type_t type) -{ + const void *src, int32_t num_vals, + pmix_data_type_t type) + { pmix_status_t rc; /* check for error */ @@ -69,7 +69,7 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, pmix_bfrop_type_info_t *info; pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_pack_buffer( %p, %p, %lu, %d )\n", - (void*)buffer, src, (long unsigned int)num_vals, (int)type); + (void*)buffer, src, (long unsigned int)num_vals, (int)type); /* Pack the declared data type */ if (PMIX_BFROP_BUFFER_FULLY_DESC == buffer->type) { @@ -94,8 +94,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * BOOL */ pmix_status_t pmix_bfrop_pack_bool(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { uint8_t *dst; int32_t i; bool *s = (bool*)src; @@ -126,8 +126,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * INT */ pmix_status_t pmix_bfrop_pack_int(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { pmix_status_t ret; /* System types need to always be described so we can properly @@ -144,8 +144,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * SIZE_T */ pmix_status_t pmix_bfrop_pack_sizet(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { pmix_status_t ret; /* System types need to always be described so we can properly @@ -161,8 +161,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * PID_T */ pmix_status_t pmix_bfrop_pack_pid(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { pmix_status_t ret; /* System types need to always be described so we can properly @@ -182,8 +182,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * BYTE, CHAR, INT8 */ pmix_status_t pmix_bfrop_pack_byte(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { char *dst; pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_pack_byte * %d\n", num_vals); @@ -206,8 +206,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * INT16 */ pmix_status_t pmix_bfrop_pack_int16(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { int32_t i; uint16_t tmp, *srctmp = (uint16_t*) src; char *dst; @@ -233,8 +233,8 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, * INT32 */ pmix_status_t pmix_bfrop_pack_int32(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { int32_t i; uint32_t tmp, *srctmp = (uint32_t*) src; char *dst; @@ -257,7 +257,7 @@ pmix_status_t pmix_bfrop_pack_buffer(pmix_buffer_t *buffer, } pmix_status_t pmix_bfrop_pack_datatype(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) + int32_t num_vals, pmix_data_type_t type) { return pmix_bfrop_pack_int32(buffer, src, num_vals, type); } @@ -266,8 +266,8 @@ pmix_status_t pmix_bfrop_pack_datatype(pmix_buffer_t *buffer, const void *src, * INT64 */ pmix_status_t pmix_bfrop_pack_int64(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { int32_t i; uint64_t tmp, tmp2; char *dst; @@ -295,31 +295,31 @@ pmix_status_t pmix_bfrop_pack_datatype(pmix_buffer_t *buffer, const void *src, * STRING */ pmix_status_t pmix_bfrop_pack_string(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ - int ret = PMIX_SUCCESS; + int32_t num_vals, pmix_data_type_t type) + { + pmix_status_t ret = PMIX_SUCCESS; int32_t i, len; char **ssrc = (char**) src; for (i = 0; i < num_vals; ++i) { if (NULL == ssrc[i]) { /* got zero-length string/NULL pointer - store NULL */ - len = 0; - if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int32(buffer, &len, 1, PMIX_INT32))) { - return ret; - } - } else { - len = (int32_t)strlen(ssrc[i]) + 1; - if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int32(buffer, &len, 1, PMIX_INT32))) { - return ret; - } - if (PMIX_SUCCESS != (ret = - pmix_bfrop_pack_byte(buffer, ssrc[i], len, PMIX_BYTE))) { - return ret; - } + len = 0; + if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int32(buffer, &len, 1, PMIX_INT32))) { + return ret; } + } else { + len = (int32_t)strlen(ssrc[i]) + 1; + if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int32(buffer, &len, 1, PMIX_INT32))) { + return ret; + } + if (PMIX_SUCCESS != (ret = + pmix_bfrop_pack_byte(buffer, ssrc[i], len, PMIX_BYTE))) { + return ret; } +} +} - return PMIX_SUCCESS; +return PMIX_SUCCESS; } /* FLOAT */ @@ -400,7 +400,7 @@ pmix_status_t pmix_bfrop_pack_time(pmix_buffer_t *buffer, const void *src, /* time_t is a system-dependent size, so cast it * to uint64_t as a generic safe size */ - for (i = 0; i < num_vals; ++i) { + for (i = 0; i < num_vals; ++i) { ui64 = (uint64_t)ssrc[i]; if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int64(buffer, &ui64, 1, PMIX_UINT64))) { return ret; @@ -410,6 +410,25 @@ pmix_status_t pmix_bfrop_pack_time(pmix_buffer_t *buffer, const void *src, return PMIX_SUCCESS; } +/* STATUS */ +pmix_status_t pmix_bfrop_pack_status(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_status_t ret = PMIX_SUCCESS; + int32_t i; + pmix_status_t *ssrc = (pmix_status_t *)src; + int32_t status; + + for (i = 0; i < num_vals; ++i) { + status = (int32_t)ssrc[i]; + if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int32(buffer, &status, 1, PMIX_INT32))) { + return ret; + } + } + + return PMIX_SUCCESS; +} + /* PACK FUNCTIONS FOR GENERIC PMIX TYPES */ static pmix_status_t pack_val(pmix_buffer_t *buffer, @@ -418,107 +437,112 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer, pmix_status_t ret; switch (p->type) { - case PMIX_BOOL: + case PMIX_BOOL: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.flag, 1, PMIX_BOOL))) { return ret; } break; - case PMIX_BYTE: + case PMIX_BYTE: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.byte, 1, PMIX_BYTE))) { return ret; } break; - case PMIX_STRING: + case PMIX_STRING: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.string, 1, PMIX_STRING))) { return ret; } break; - case PMIX_SIZE: + case PMIX_SIZE: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.size, 1, PMIX_SIZE))) { return ret; } break; - case PMIX_PID: + case PMIX_PID: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.pid, 1, PMIX_PID))) { return ret; } break; - case PMIX_INT: + case PMIX_INT: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.integer, 1, PMIX_INT))) { return ret; } break; - case PMIX_INT8: + case PMIX_INT8: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.int8, 1, PMIX_INT8))) { return ret; } break; - case PMIX_INT16: + case PMIX_INT16: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.int16, 1, PMIX_INT16))) { return ret; } break; - case PMIX_INT32: + case PMIX_INT32: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.int32, 1, PMIX_INT32))) { return ret; } break; - case PMIX_INT64: + case PMIX_INT64: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.int64, 1, PMIX_INT64))) { return ret; } break; - case PMIX_UINT: + case PMIX_UINT: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.uint, 1, PMIX_UINT))) { return ret; } break; - case PMIX_UINT8: + case PMIX_UINT8: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.uint8, 1, PMIX_UINT8))) { return ret; } break; - case PMIX_UINT16: + case PMIX_UINT16: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.uint16, 1, PMIX_UINT16))) { return ret; } break; - case PMIX_UINT32: + case PMIX_UINT32: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.uint32, 1, PMIX_UINT32))) { return ret; } break; - case PMIX_UINT64: + case PMIX_UINT64: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.uint64, 1, PMIX_UINT64))) { return ret; } break; - case PMIX_FLOAT: + case PMIX_FLOAT: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.fval, 1, PMIX_FLOAT))) { return ret; } break; - case PMIX_DOUBLE: + case PMIX_DOUBLE: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.dval, 1, PMIX_DOUBLE))) { return ret; } break; - case PMIX_TIMEVAL: + case PMIX_TIMEVAL: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.tv, 1, PMIX_TIMEVAL))) { return ret; } break; - case PMIX_INFO_ARRAY: + case PMIX_STATUS: + if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.status, 1, PMIX_STATUS))) { + return ret; + } + break; + case PMIX_INFO_ARRAY: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.array, 1, PMIX_INFO_ARRAY))) { return ret; } break; - case PMIX_BYTE_OBJECT: + case PMIX_BYTE_OBJECT: if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_buffer(buffer, &p->data.bo, 1, PMIX_BYTE_OBJECT))) { return ret; } break; - default: + default: pmix_output(0, "PACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)p->type); return PMIX_ERROR; } @@ -529,8 +553,8 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer, * PMIX_VALUE */ pmix_status_t pmix_bfrop_pack_value(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ + int32_t num_vals, pmix_data_type_t type) + { pmix_value_t *ptr; int32_t i; pmix_status_t ret; @@ -568,6 +592,10 @@ pmix_status_t pmix_bfrop_pack_info(pmix_buffer_t *buffer, const void *src, if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_string(buffer, &foo, 1, PMIX_STRING))) { return ret; } + /* pack required flag */ + if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_bool(buffer, &info[i].required, 1, PMIX_BOOL))) { + return ret; + } /* pack the type */ if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int(buffer, &info[i].value.type, 1, PMIX_INT))) { return ret; @@ -581,7 +609,7 @@ pmix_status_t pmix_bfrop_pack_info(pmix_buffer_t *buffer, const void *src, } pmix_status_t pmix_bfrop_pack_pdata(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) + int32_t num_vals, pmix_data_type_t type) { pmix_pdata_t *pdata; int32_t i; @@ -637,7 +665,7 @@ pmix_status_t pmix_bfrop_pack_buf(pmix_buffer_t *buffer, const void *src, } pmix_status_t pmix_bfrop_pack_proc(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) + int32_t num_vals, pmix_data_type_t type) { pmix_proc_t *proc; int32_t i; @@ -789,28 +817,28 @@ pmix_status_t pmix_bfrop_pack_topo(pmix_buffer_t *buffer, const void *src, /* get the available support - hwloc unfortunately does * not include this info in its xml export! */ - support = (struct hwloc_topology_support*)hwloc_topology_get_support(t); + support = (struct hwloc_topology_support*)hwloc_topology_get_support(t); /* pack the discovery support */ - if (PMIX_SUCCESS != (rc = pmix_bfrop_pack_byte(buffer, support->discovery, - sizeof(struct hwloc_topology_discovery_support), - PMIX_BYTE))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop_pack_byte(buffer, support->discovery, + sizeof(struct hwloc_topology_discovery_support), + PMIX_BYTE))) { return rc; - } - /* pack the cpubind support */ - if (PMIX_SUCCESS != (rc = pmix_bfrop_pack_byte(buffer, support->cpubind, - sizeof(struct hwloc_topology_cpubind_support), - PMIX_BYTE))) { - return rc; - } - /* pack the membind support */ - if (PMIX_SUCCESS != (rc = pmix_bfrop_pack_byte(buffer, support->membind, - sizeof(struct hwloc_topology_membind_support), - PMIX_BYTE))) { - return rc; - } } + /* pack the cpubind support */ + if (PMIX_SUCCESS != (rc = pmix_bfrop_pack_byte(buffer, support->cpubind, + sizeof(struct hwloc_topology_cpubind_support), + PMIX_BYTE))) { + return rc; +} + /* pack the membind support */ +if (PMIX_SUCCESS != (rc = pmix_bfrop_pack_byte(buffer, support->membind, + sizeof(struct hwloc_topology_membind_support), + PMIX_BYTE))) { + return rc; +} +} - return PMIX_SUCCESS; +return PMIX_SUCCESS; } #endif @@ -862,3 +890,13 @@ pmix_status_t pmix_bfrop_pack_bo(pmix_buffer_t *buffer, const void *src, } return PMIX_SUCCESS; } + +pmix_status_t pmix_bfrop_pack_ptr(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + uint8_t foo=1; + /* it obviously makes no sense to pack a pointer and + * send it somewhere else, so we just pack a sentinel */ + return pmix_bfrop_pack_byte(buffer, &foo, 1, PMIX_UINT8); +} + diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/print.c b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/print.c similarity index 92% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/print.c rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/print.c index fd3db0982f..c4730dd19c 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/print.c +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/print.c @@ -30,10 +30,12 @@ #include #endif +#include "src/util/error.h" + #include "src/buffer_ops/internal.h" -pmix_status_t pmix_bfrop_print(char **output, char *prefix, void *src, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_print(char **output, char *prefix, void *src, pmix_data_type_t type) + { pmix_bfrop_type_info_t *info; /* check for error */ @@ -53,8 +55,8 @@ pmix_status_t pmix_bfrop_print(char **output, char *prefix, void *src, pmix_data /* * STANDARD PRINT FUNCTIONS FOR SYSTEM TYPES */ -pmix_status_t pmix_bfrop_print_bool(char **output, char *prefix, bool *src, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_print_bool(char **output, char *prefix, bool *src, pmix_data_type_t type) + { char *prefx; /* deal with NULL prefix */ @@ -715,14 +717,50 @@ pmix_status_t pmix_bfrop_print_timeval(char **output, char *prefix, return PMIX_SUCCESS; } +pmix_status_t pmix_bfrop_print_status(char **output, char *prefix, + pmix_status_t *src, pmix_data_type_t type) +{ + char *prefx; + + /* deal with NULL prefix */ + if (NULL == prefix) { + if (0 > asprintf(&prefx, " ")) { + return PMIX_ERR_NOMEM; + } + } else { + prefx = prefix; + } + + /* if src is NULL, just print data type and return */ + if (NULL == src) { + if (0 > asprintf(output, "%sData type: PMIX_STATUS\tValue: NULL pointer", prefx)) { + return PMIX_ERR_NOMEM; + } + if (prefx != prefix) { + free(prefx); + } + return PMIX_SUCCESS; + } + + if (0 > asprintf(output, "%sData type: PMIX_STATUS\tValue: %s", prefx, PMIx_Error_string(*src))) { + return PMIX_ERR_NOMEM; + } + if (prefx != prefix) { + free(prefx); + } + + return PMIX_SUCCESS; +} + + /* PRINT FUNCTIONS FOR GENERIC PMIX TYPES */ /* * PMIX_VALUE */ -pmix_status_t pmix_bfrop_print_value(char **output, char *prefix, - pmix_value_t *src, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_print_value(char **output, char *prefix, + pmix_value_t *src, pmix_data_type_t type) + { char *prefx; int rc; @@ -815,7 +853,10 @@ pmix_status_t pmix_bfrop_print_value(char **output, char *prefix, rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_TIMEVAL\tValue: %ld.%06ld", prefx, (long)src->data.tv.tv_sec, (long)src->data.tv.tv_usec); break; - + case PMIX_STATUS: + rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_STATUS\tValue: %s", prefx, + PMIx_Error_string(src->data.status)); + break; default: rc = asprintf(output, "%sPMIX_VALUE: Data type: UNKNOWN\tValue: UNPRINTABLE", prefx); break; @@ -836,8 +877,8 @@ pmix_status_t pmix_bfrop_print_info(char **output, char *prefix, int rc; pmix_bfrop_print_value(&tmp, NULL, &src->value, PMIX_VALUE); - rc = asprintf(output, "%sKEY: %s %s", prefix, src->key, - (NULL == tmp) ? "PMIX_VALUE: NULL" : tmp); + rc = asprintf(output, "%sKEY: %s REQD: %s %s", prefix, src->key, + src->required ? "Y" : "N", (NULL == tmp) ? "PMIX_VALUE: NULL" : tmp); if (NULL != tmp) { free(tmp); } @@ -848,7 +889,7 @@ pmix_status_t pmix_bfrop_print_info(char **output, char *prefix, } pmix_status_t pmix_bfrop_print_pdata(char **output, char *prefix, - pmix_pdata_t *src, pmix_data_type_t type) + pmix_pdata_t *src, pmix_data_type_t type) { char *tmp1, *tmp2; int rc; @@ -882,7 +923,7 @@ pmix_status_t pmix_bfrop_print_app(char **output, char *prefix, } pmix_status_t pmix_bfrop_print_proc(char **output, char *prefix, - pmix_proc_t *src, pmix_data_type_t type) + pmix_proc_t *src, pmix_data_type_t type) { char *prefx; @@ -973,7 +1014,7 @@ static void print_hwloc_obj(char **output, char *prefix, /* print the cpusets - apparently, some new HWLOC types don't * have cpusets, so protect ourselves here */ - if (NULL != obj->cpuset) { + if (NULL != obj->cpuset) { hwloc_bitmap_snprintf(string, PMIX_HWLOC_MAX_STRING, obj->cpuset); if (0 > asprintf(&tmp2, "%s%sCpuset: %s", tmp, pfx, string)) { free(tmp); @@ -1137,3 +1178,27 @@ pmix_status_t pmix_bfrop_print_bo(char **output, char *prefix, return PMIX_SUCCESS; } +pmix_status_t pmix_bfrop_print_ptr(char **output, char *prefix, + void *src, pmix_data_type_t type) +{ + char *prefx; + + /* deal with NULL prefix */ + if (NULL == prefix) { + if (0 > asprintf(&prefx, " ")) { + return PMIX_ERR_NOMEM; + } + } else { + prefx = prefix; + } + + if (0 > asprintf(output, "%sData type: PMIX_POINTER\tAddress: %p", prefx, src)) { + return PMIX_ERR_NOMEM; + } + if (prefx != prefix) { + free(prefx); + } + + return PMIX_SUCCESS; +} + diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/types.h b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/types.h similarity index 94% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/types.h rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/types.h index 762b9c1531..cb9288131a 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/types.h +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/types.h @@ -78,7 +78,7 @@ typedef struct { including overhead -- packed in the buffer) */ size_t bytes_used; } pmix_buffer_t; -PMIX_DECLSPEC PMIX_CLASS_DECLARATION (pmix_buffer_t); +PMIX_CLASS_DECLARATION (pmix_buffer_t); /* these classes are required by the regex code shared * between the client and server implementations - it @@ -88,7 +88,7 @@ typedef struct { int start; int cnt; } pmix_regex_range_t; -PMIX_DECLSPEC PMIX_CLASS_DECLARATION(pmix_regex_range_t); +PMIX_CLASS_DECLARATION(pmix_regex_range_t); typedef struct { /* list object */ @@ -98,7 +98,7 @@ typedef struct { int num_digits; pmix_list_t ranges; } pmix_regex_value_t; -PMIX_DECLSPEC PMIX_CLASS_DECLARATION(pmix_regex_value_t); +PMIX_CLASS_DECLARATION(pmix_regex_value_t); END_C_DECLS diff --git a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/unpack.c b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/unpack.c similarity index 88% rename from opal/mca/pmix/pmix114/pmix/src/buffer_ops/unpack.c rename to opal/mca/pmix/pmix2x/pmix/src/buffer_ops/unpack.c index 8be78b7659..f775348e96 100644 --- a/opal/mca/pmix/pmix114/pmix/src/buffer_ops/unpack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/buffer_ops/unpack.c @@ -33,9 +33,9 @@ #include "src/buffer_ops/types.h" #include "src/buffer_ops/internal.h" -pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dst, int32_t *num_vals, - pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dst, int32_t *num_vals, + pmix_data_type_t type) + { pmix_status_t rc, ret; int32_t local_num, n=1; pmix_data_type_t local_type; @@ -48,9 +48,9 @@ pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dst, int32_t *num_v /* if user provides a zero for num_vals, then there is no storage allocated * so return an appropriate error */ - if (0 == *num_vals) { + if (0 == *num_vals) { pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack: inadequate space ( %p, %p, %lu, %d )\n", - (void*)buffer, dst, (long unsigned int)*num_vals, (int)type); + (void*)buffer, dst, (long unsigned int)*num_vals, (int)type); return PMIX_ERR_UNPACK_INADEQUATE_SPACE; } @@ -63,7 +63,7 @@ pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dst, int32_t *num_v * NOT completely safe. This is true for ALL unpack functions, not just * int32_t as used here. */ - if (PMIX_BFROP_BUFFER_FULLY_DESC == buffer->type) { + if (PMIX_BFROP_BUFFER_FULLY_DESC == buffer->type) { if (PMIX_SUCCESS != (rc = pmix_bfrop_get_data_type(buffer, &local_type))) { *num_vals = 0; /* don't error log here as the user may be unpacking past @@ -71,31 +71,31 @@ pmix_status_t pmix_bfrop_unpack(pmix_buffer_t *buffer, void *dst, int32_t *num_v return rc; } if (PMIX_INT32 != local_type) { /* if the length wasn't first, then error */ - *num_vals = 0; - return PMIX_ERR_UNPACK_FAILURE; - } - } - - n=1; - if (PMIX_SUCCESS != (rc = pmix_bfrop_unpack_int32(buffer, &local_num, &n, PMIX_INT32))) { *num_vals = 0; + return PMIX_ERR_UNPACK_FAILURE; + } +} + +n=1; +if (PMIX_SUCCESS != (rc = pmix_bfrop_unpack_int32(buffer, &local_num, &n, PMIX_INT32))) { + *num_vals = 0; /* don't error log here as the user may be unpacking past * the end of the buffer, which isn't necessarily an error */ - return rc; - } + return rc; +} - pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack: found %d values for %d provided storage", - local_num, *num_vals); +pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack: found %d values for %d provided storage", + local_num, *num_vals); /** if the storage provided is inadequate, set things up * to unpack as much as we can and to return an error code * indicating that everything was not unpacked - the buffer * is left in a state where it can not be further unpacked. */ - if (local_num > *num_vals) { + if (local_num > *num_vals) { local_num = *num_vals; pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack: inadequate space ( %p, %p, %lu, %d )\n", - (void*)buffer, dst, (long unsigned int)*num_vals, (int)type); + (void*)buffer, dst, (long unsigned int)*num_vals, (int)type); ret = PMIX_ERR_UNPACK_INADEQUATE_SPACE; } else { /** enough or more than enough storage */ *num_vals = local_num; /** let the user know how many we actually unpacked */ @@ -119,7 +119,7 @@ pmix_status_t pmix_bfrop_unpack_buffer(pmix_buffer_t *buffer, void *dst, int32_t pmix_bfrop_type_info_t *info; pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack_buffer( %p, %p, %lu, %d )\n", - (void*)buffer, dst, (long unsigned int)*num_vals, (int)type); + (void*)buffer, dst, (long unsigned int)*num_vals, (int)type); /** Unpack the declared data type */ if (PMIX_BFROP_BUFFER_FULLY_DESC == buffer->type) { @@ -148,9 +148,9 @@ pmix_status_t pmix_bfrop_unpack_buffer(pmix_buffer_t *buffer, void *dst, int32_t /* * BOOL */ -pmix_status_t pmix_bfrop_unpack_bool(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_unpack_bool(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) + { int32_t i; uint8_t *src; bool *dst; @@ -182,9 +182,9 @@ pmix_status_t pmix_bfrop_unpack_bool(pmix_buffer_t *buffer, void *dest, /* * INT */ -pmix_status_t pmix_bfrop_unpack_int(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_unpack_int(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) + { pmix_status_t ret; pmix_data_type_t remote_type; @@ -208,9 +208,9 @@ pmix_status_t pmix_bfrop_unpack_int(pmix_buffer_t *buffer, void *dest, /* * SIZE_T */ -pmix_status_t pmix_bfrop_unpack_sizet(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_unpack_sizet(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) + { pmix_status_t ret; pmix_data_type_t remote_type; @@ -234,9 +234,9 @@ pmix_status_t pmix_bfrop_unpack_sizet(pmix_buffer_t *buffer, void *dest, /* * PID_T */ -pmix_status_t pmix_bfrop_unpack_pid(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_unpack_pid(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) + { pmix_status_t ret; pmix_data_type_t remote_type; @@ -263,9 +263,9 @@ pmix_status_t pmix_bfrop_unpack_pid(pmix_buffer_t *buffer, void *dest, /* * BYTE, CHAR, INT8 */ -pmix_status_t pmix_bfrop_unpack_byte(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ + pmix_status_t pmix_bfrop_unpack_byte(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) + { pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack_byte * %d\n", (int)*num_vals); /* check to see if there's enough data in buffer */ if (pmix_bfrop_too_small(buffer, *num_vals)) { @@ -328,7 +328,7 @@ pmix_status_t pmix_bfrop_unpack_int32(pmix_buffer_t *buffer, void *dest, } pmix_status_t pmix_bfrop_unpack_datatype(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) + int32_t *num_vals, pmix_data_type_t type) { return pmix_bfrop_unpack_int32(buffer, dest, num_vals, type); } @@ -357,7 +357,7 @@ pmix_status_t pmix_bfrop_unpack_int64(pmix_buffer_t *buffer, void *dest, } pmix_status_t pmix_bfrop_unpack_string(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) + int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; int32_t i, len, n=1; @@ -368,19 +368,19 @@ pmix_status_t pmix_bfrop_unpack_string(pmix_buffer_t *buffer, void *dest, return ret; } if (0 == len) { /* zero-length string - unpack the NULL */ - sdest[i] = NULL; - } else { - sdest[i] = (char*)malloc(len); - if (NULL == sdest[i]) { - return PMIX_ERR_OUT_OF_RESOURCE; - } - if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_byte(buffer, sdest[i], &len, PMIX_BYTE))) { - return ret; - } + sdest[i] = NULL; + } else { + sdest[i] = (char*)malloc(len); + if (NULL == sdest[i]) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_byte(buffer, sdest[i], &len, PMIX_BYTE))) { + return ret; } } +} - return PMIX_SUCCESS; +return PMIX_SUCCESS; } pmix_status_t pmix_bfrop_unpack_float(pmix_buffer_t *buffer, void *dest, @@ -482,9 +482,9 @@ pmix_status_t pmix_bfrop_unpack_time(pmix_buffer_t *buffer, void *dest, * to uint64_t as a generic safe size */ - pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack_time * %d\n", (int)*num_vals); + pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack_time * %d\n", (int)*num_vals); /* check to see if there's enough data in buffer */ - if (pmix_bfrop_too_small(buffer, (*num_vals)*(sizeof(uint64_t)))) { + if (pmix_bfrop_too_small(buffer, (*num_vals)*(sizeof(uint64_t)))) { return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; } @@ -501,119 +501,138 @@ pmix_status_t pmix_bfrop_unpack_time(pmix_buffer_t *buffer, void *dest, } +pmix_status_t pmix_bfrop_unpack_status(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_output_verbose(20, pmix_globals.debug_output, "pmix_bfrop_unpack_status * %d\n", (int)*num_vals); + /* check to see if there's enough data in buffer */ + if (pmix_bfrop_too_small(buffer, (*num_vals)*(sizeof(pmix_status_t)))) { + return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; + } + + /* unpack the data */ + return pmix_bfrop_unpack_int32(buffer, dest, num_vals, PMIX_INT32); +} + + /* UNPACK FUNCTIONS FOR GENERIC PMIX TYPES */ /* * PMIX_VALUE */ -static pmix_status_t unpack_val(pmix_buffer_t *buffer, pmix_value_t *val) -{ + static pmix_status_t unpack_val(pmix_buffer_t *buffer, pmix_value_t *val) + { int32_t m; pmix_status_t ret; m = 1; switch (val->type) { - case PMIX_BOOL: + case PMIX_BOOL: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.flag, &m, PMIX_BOOL))) { return ret; } break; - case PMIX_BYTE: + case PMIX_BYTE: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.byte, &m, PMIX_BYTE))) { return ret; } break; - case PMIX_STRING: + case PMIX_STRING: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.string, &m, PMIX_STRING))) { return ret; } break; - case PMIX_SIZE: + case PMIX_SIZE: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.size, &m, PMIX_SIZE))) { return ret; } break; - case PMIX_PID: + case PMIX_PID: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.pid, &m, PMIX_PID))) { return ret; } break; - case PMIX_INT: + case PMIX_INT: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.integer, &m, PMIX_INT))) { return ret; } break; - case PMIX_INT8: + case PMIX_INT8: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.int8, &m, PMIX_INT8))) { return ret; } break; - case PMIX_INT16: + case PMIX_INT16: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.int16, &m, PMIX_INT16))) { return ret; } break; - case PMIX_INT32: + case PMIX_INT32: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.int32, &m, PMIX_INT32))) { return ret; } break; - case PMIX_INT64: + case PMIX_INT64: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.int64, &m, PMIX_INT64))) { return ret; } break; - case PMIX_UINT: + case PMIX_UINT: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.uint, &m, PMIX_UINT))) { return ret; } break; - case PMIX_UINT8: + case PMIX_UINT8: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.uint8, &m, PMIX_UINT8))) { return ret; } break; - case PMIX_UINT16: + case PMIX_UINT16: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.uint16, &m, PMIX_UINT16))) { return ret; } break; - case PMIX_UINT32: + case PMIX_UINT32: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.uint32, &m, PMIX_UINT32))) { return ret; } break; - case PMIX_UINT64: + case PMIX_UINT64: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.uint64, &m, PMIX_UINT64))) { return ret; } break; - case PMIX_FLOAT: + case PMIX_FLOAT: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.fval, &m, PMIX_FLOAT))) { return ret; } break; - case PMIX_DOUBLE: + case PMIX_DOUBLE: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.dval, &m, PMIX_DOUBLE))) { return ret; } break; - case PMIX_TIMEVAL: + case PMIX_TIMEVAL: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.tv, &m, PMIX_TIMEVAL))) { return ret; } break; - case PMIX_INFO_ARRAY: + case PMIX_STATUS: + if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.status, &m, PMIX_STATUS))) { + return ret; + } + break; + case PMIX_INFO_ARRAY: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.array, &m, PMIX_INFO_ARRAY))) { return ret; } break; - case PMIX_BYTE_OBJECT: + case PMIX_BYTE_OBJECT: if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_buffer(buffer, &val->data.bo, &m, PMIX_BYTE_OBJECT))) { return ret; } break; - default: + default: pmix_output(0, "UNPACK-PMIX-VALUE: UNSUPPORTED TYPE"); return PMIX_ERROR; } @@ -673,11 +692,16 @@ pmix_status_t pmix_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, } (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); + /* unpack the required flag */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_bool(buffer, &ptr[i].required, &m, PMIX_BOOL))) { + return ret; + } /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to * avoid the malloc */ - m=1; - if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &ptr[i].value.type, &m, PMIX_INT))) { + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &ptr[i].value.type, &m, PMIX_INT))) { return ret; } pmix_output_verbose(20, pmix_globals.debug_output, @@ -691,7 +715,7 @@ pmix_status_t pmix_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, } pmix_status_t pmix_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) + int32_t *num_vals, pmix_data_type_t type) { pmix_pdata_t *ptr; int32_t i, n, m; @@ -725,8 +749,8 @@ pmix_status_t pmix_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to * avoid the malloc */ - m=1; - if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &ptr[i].value.type, &m, PMIX_INT))) { + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &ptr[i].value.type, &m, PMIX_INT))) { return ret; } pmix_output_verbose(20, pmix_globals.debug_output, @@ -997,7 +1021,7 @@ pmix_status_t pmix_bfrop_unpack_topo(pmix_buffer_t *buffer, void *dest, /* since we are loading this from an external source, we have to * explicitly set a flag so hwloc sets things up correctly */ - if (0 != hwloc_topology_set_flags(t, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM | HWLOC_TOPOLOGY_FLAG_IO_DEVICES)) { + if (0 != hwloc_topology_set_flags(t, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM | HWLOC_TOPOLOGY_FLAG_IO_DEVICES)) { free(xmlbuffer); rc = PMIX_ERROR; hwloc_topology_destroy(t); @@ -1017,9 +1041,9 @@ pmix_status_t pmix_bfrop_unpack_topo(pmix_buffer_t *buffer, void *dest, /* get the available support - hwloc unfortunately does * not include this info in its xml import! */ - support = (struct hwloc_topology_support*)hwloc_topology_get_support(t); - cnt = sizeof(struct hwloc_topology_discovery_support); - if (PMIX_SUCCESS != (rc = pmix_bfrop_unpack_byte(buffer, support->discovery, &cnt, PMIX_BYTE))) { + support = (struct hwloc_topology_support*)hwloc_topology_get_support(t); + cnt = sizeof(struct hwloc_topology_discovery_support); + if (PMIX_SUCCESS != (rc = pmix_bfrop_unpack_byte(buffer, support->discovery, &cnt, PMIX_BYTE))) { goto cleanup; } cnt = sizeof(struct hwloc_topology_cpubind_support); @@ -1038,7 +1062,7 @@ pmix_status_t pmix_bfrop_unpack_topo(pmix_buffer_t *buffer, void *dest, j++; } - cleanup: + cleanup: *num_vals = j; return rc; } @@ -1077,7 +1101,7 @@ pmix_status_t pmix_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, pmix_status_t pmix_bfrop_unpack_persist(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) + int32_t *num_vals, pmix_data_type_t type) { return pmix_bfrop_unpack_int(buffer, dest, num_vals, PMIX_INT); } @@ -1112,3 +1136,15 @@ pmix_status_t pmix_bfrop_unpack_bo(pmix_buffer_t *buffer, void *dest, } return PMIX_SUCCESS; } + +pmix_status_t pmix_bfrop_unpack_ptr(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + uint8_t foo=1; + int32_t cnt=1; + + /* it obviously makes no sense to pack a pointer and + * send it somewhere else, so we just unpack the sentinel */ + return pmix_bfrop_unpack_byte(buffer, &foo, &cnt, PMIX_UINT8); +} + diff --git a/opal/mca/pmix/pmix114/pmix/src/class/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/class/Makefile.am similarity index 79% rename from opal/mca/pmix/pmix114/pmix/src/class/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/class/Makefile.am index a173ff46d9..d13dad2d5d 100644 --- a/opal/mca/pmix/pmix114/pmix/src/class/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/class/Makefile.am @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2013-2015 Intel, Inc. All rights reserved +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved # $COPYRIGHT$ # # Additional copyrights may follow @@ -26,10 +26,14 @@ headers += \ src/class/pmix_object.h \ src/class/pmix_list.h \ src/class/pmix_pointer_array.h \ - src/class/pmix_hash_table.h + src/class/pmix_hash_table.h \ + src/class/pmix_hotel.h \ + src/class/pmix_ring_buffer.h sources += \ src/class/pmix_object.c \ src/class/pmix_list.c \ src/class/pmix_pointer_array.c \ - src/class/pmix_hash_table.c + src/class/pmix_hash_table.c \ + src/class/pmix_hotel.c \ + src/class/pmix_ring_buffer.c diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_hash_table.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hash_table.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_hash_table.c rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_hash_table.c diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_hash_table.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hash_table.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_hash_table.h rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_hash_table.h diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c new file mode 100644 index 0000000000..9efcdaf8cc --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#include + +#include PMIX_EVENT_HEADER +#include "src/class/pmix_hotel.h" + + +static void local_eviction_callback(int fd, short flags, void *arg) +{ + pmix_hotel_room_eviction_callback_arg_t *eargs = + (pmix_hotel_room_eviction_callback_arg_t*) arg; + void *occupant = eargs->hotel->rooms[eargs->room_num].occupant; + + /* Remove the occurpant from the room. + + Do not change this logic without also changing the same logic + in pmix_hotel_checkout() and + pmix_hotel_checkout_and_return_occupant(). */ + pmix_hotel_t *hotel = eargs->hotel; + pmix_hotel_room_t *room = &(hotel->rooms[eargs->room_num]); + room->occupant = NULL; + hotel->last_unoccupied_room++; + assert(hotel->last_unoccupied_room < hotel->num_rooms); + hotel->unoccupied_rooms[hotel->last_unoccupied_room] = eargs->room_num; + + /* Invoke the user callback to tell them that they were evicted */ + hotel->evict_callback_fn(hotel, + eargs->room_num, + occupant); +} + + +int pmix_hotel_init(pmix_hotel_t *h, int num_rooms, + pmix_event_base_t *evbase, + uint32_t eviction_timeout, + int eviction_event_priority, + pmix_hotel_eviction_callback_fn_t evict_callback_fn) +{ + int i; + + /* Bozo check */ + if (num_rooms <= 0 || + NULL == evict_callback_fn) { + return PMIX_ERR_BAD_PARAM; + } + + h->num_rooms = num_rooms; + h->evbase = evbase; + h->eviction_timeout.tv_usec = eviction_timeout % 1000000; + h->eviction_timeout.tv_sec = eviction_timeout / 1000000; + h->evict_callback_fn = evict_callback_fn; + h->rooms = (pmix_hotel_room_t*)malloc(num_rooms * sizeof(pmix_hotel_room_t)); + if (NULL != evict_callback_fn) { + h->eviction_args = + (pmix_hotel_room_eviction_callback_arg_t*)malloc(num_rooms * sizeof(pmix_hotel_room_eviction_callback_arg_t)); + } + h->unoccupied_rooms = (int*) malloc(num_rooms * sizeof(int)); + h->last_unoccupied_room = num_rooms - 1; + + for (i = 0; i < num_rooms; ++i) { + /* Mark this room as unoccupied */ + h->rooms[i].occupant = NULL; + + /* Setup this room in the unoccupied index array */ + h->unoccupied_rooms[i] = i; + + /* Setup the eviction callback args */ + h->eviction_args[i].hotel = h; + h->eviction_args[i].room_num = i; + + /* Create this room's event (but don't add it) */ + if (NULL != h->evbase) { + event_assign(&(h->rooms[i].eviction_timer_event), + h->evbase, + -1, 0, local_eviction_callback, + &(h->eviction_args[i])); + } + } + + return PMIX_SUCCESS; +} + +static void constructor(pmix_hotel_t *h) +{ + h->num_rooms = 0; + h->evbase = NULL; + h->eviction_timeout.tv_sec = 0; + h->eviction_timeout.tv_usec = 0; + h->evict_callback_fn = NULL; + h->rooms = NULL; + h->eviction_args = NULL; + h->unoccupied_rooms = NULL; + h->last_unoccupied_room = -1; +} + +static void destructor(pmix_hotel_t *h) +{ + int i; + + /* Go through all occupied rooms and destroy their events */ + if (NULL != h->evbase) { + for (i = 0; i < h->num_rooms; ++i) { + if (NULL != h->rooms[i].occupant) { + event_del(&(h->rooms[i].eviction_timer_event)); + } + } + } + + if (NULL != h->rooms) { + free(h->rooms); + } + if (NULL != h->eviction_args) { + free(h->eviction_args); + } + if (NULL != h->unoccupied_rooms) { + free(h->unoccupied_rooms); + } +} + +PMIX_CLASS_INSTANCE(pmix_hotel_t, + pmix_object_t, + constructor, + destructor); diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h new file mode 100644 index 0000000000..c2e6f0f4af --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file + * + * This file provides a "hotel" class: + * + * - A hotel has a fixed number of rooms (i.e., storage slots) + * - An arbitrary data pointer can check into an empty room at any time + * - The occupant of a room can check out at any time + * - Optionally, the occupant of a room can be forcibly evicted at a + * given time (i.e., when an pmix timer event expires). + * - The hotel has finite occupancy; if you try to checkin a new + * occupant and the hotel is already full, it will gracefully fail + * to checkin. + * + * One use case for this class is for ACK-based network retransmission + * schemes (NACK-based retransmission schemes probably can use + * pmix_ring_buffer). + * + * For ACK-based retransmission schemes, a hotel might be used + * something like this: + * + * - when a message is sent, check it in to a hotel with a timer + * - if an ACK is received, check it out of the hotel (which also cancels + * the timer) + * - if an ACK isn't received in time, the timer will expire and the + * upper layer will get a callback with the message + * - if an ACK is received late (i.e., after its timer has expired), + * then checkout will gracefully fail + * + * Note that this class intentionally provides pretty minimal + * functionality. It is intended to be used in performance-critical + * code paths -- extra functionality would simply add latency. + * + * There is an pmix_hotel_init() function to create a hotel, but no + * corresponding finalize; the destructor will handle all finalization + * issues. Note that when a hotel is destroyed, it will delete all + * pending events from the event base (i.e., all pending eviction + * callbacks); no further eviction callbacks will be invoked. + */ + +#ifndef PMIX_HOTEL_H +#define PMIX_HOTEL_H + +#include +#include "src/include/types.h" +#include "src/include/prefetch.h" +#include "pmix/pmix_common.h" +#include "src/class/pmix_object.h" +#include PMIX_EVENT_HEADER + +#include "src/util/output.h" + +BEGIN_C_DECLS + +struct pmix_hotel_t; + +/* User-supplied function to be invoked when an occupant is evicted. */ +typedef void (*pmix_hotel_eviction_callback_fn_t)(struct pmix_hotel_t *hotel, + int room_num, + void *occupant); + +/* Note that this is an internal data structure; it is not part of the + public pmix_hotel interface. Public consumers of pmix_hotel + shouldn't need to use this struct at all (we only have it here in + this .h file because some functions are inlined for speed, and need + to get to the internals of this struct). + + The room struct should be as small as possible to be cache + friendly. Specifically: it would be great if multiple rooms could + fit in a single cache line because we'll always allocate a + contiguous set of rooms in an array. */ +typedef struct { + void *occupant; + pmix_event_t eviction_timer_event; +} pmix_hotel_room_t; + +/* Note that this is an internal data structure; it is not part of the + public pmix_hotel interface. Public consumers of pmix_hotel + shouldn't need to use this struct at all (we only have it here in + this .h file because some functions are inlined for speed, and need + to get to the internals of this struct). + + Use a unique struct for holding the arguments for eviction + callbacks. We *could* make the to-be-evicted pmix_hotel_room_t + instance as the argument, but we don't, for 2 reasons: + + 1. We want as many pmix_hotel_room_t's to fit in a cache line as + possible (i.e., to be as cache-friendly as possible). The + common/fast code path only needs to access the data in the + pmix_hotel_room_t (and not the callback argument data). + + 2. Evictions will be uncommon, so we don't mind penalizing them a + bit by making the data be in a separate cache line. +*/ +typedef struct { + struct pmix_hotel_t *hotel; + int room_num; +} pmix_hotel_room_eviction_callback_arg_t; + +typedef struct pmix_hotel_t { + /* make this an object */ + pmix_object_t super; + + /* Max number of rooms in the hotel */ + int num_rooms; + + /* event base to be used for eviction timeout */ + pmix_event_base_t *evbase; + struct timeval eviction_timeout; + pmix_hotel_eviction_callback_fn_t evict_callback_fn; + + /* All rooms in this hotel */ + pmix_hotel_room_t *rooms; + + /* Separate array for all the eviction callback arguments (see + rationale above for why this is a separate array) */ + pmix_hotel_room_eviction_callback_arg_t *eviction_args; + + /* All currently unoccupied rooms in this hotel (not necessarily + in any particular order) */ + int *unoccupied_rooms; + int last_unoccupied_room; +} pmix_hotel_t; +PMIX_CLASS_DECLARATION(pmix_hotel_t); + +/** + * Initialize the hotel. + * + * @param hotel Pointer to a hotel (IN) + * @param num_rooms The total number of rooms in the hotel (IN) + * @param evbase Pointer to event base used for eviction timeout + * @param eviction_timeout Max length of a stay at the hotel before + * the eviction callback is invoked (in microseconds) + * @param eviction_event_priority Event lib priority for the eviction timeout + * @param evict_callback_fn Callback function invoked if an occupant + * does not check out before the eviction_timeout. + * + * NOTE: If the callback function is NULL, then no eviction timer + * will be set - occupants will remain checked into the hotel until + * explicitly checked out. + * + * Also note: the eviction_callback_fn should absolutely not call any + * of the hotel checkout functions. Specifically: the occupant has + * already been ("forcibly") checked out *before* the + * eviction_callback_fn is invoked. + * + * @return PMIX_SUCCESS if all initializations were succesful. Otherwise, + * the error indicate what went wrong in the function. + */ +PMIX_DECLSPEC int pmix_hotel_init(pmix_hotel_t *hotel, int num_rooms, + pmix_event_base_t *evbase, + uint32_t eviction_timeout, + int eviction_event_priority, + pmix_hotel_eviction_callback_fn_t evict_callback_fn); + +/** + * Check in an occupant to the hotel. + * + * @param hotel Pointer to hotel (IN) + * @param occupant Occupant to check in (opaque to the hotel) (IN) + * @param room The room number that identifies this occupant in the + * hotel (OUT). + * + * If there is room in the hotel, the occupant is checked in and the + * timer for that occupant is started. The occupant's room is + * returned in the "room" param. + * + * Note that once a room's checkout_expire timer expires, the occupant + * is forcibly checked out, and then the eviction callback is invoked. + * + * @return PMIX_SUCCESS if the occupant is successfully checked in, + * and the room parameter will contain a valid value. + * @return PMIX_ERR_TEMP_OUT_OF_RESOURCE is the hotel is full. Try + * again later. + */ +static inline int pmix_hotel_checkin(pmix_hotel_t *hotel, + void *occupant, + int *room_num) +{ + pmix_hotel_room_t *room; + + /* Do we have any rooms available? */ + if (PMIX_UNLIKELY(hotel->last_unoccupied_room < 0)) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + + /* Put this occupant into the first empty room that we have */ + *room_num = hotel->unoccupied_rooms[hotel->last_unoccupied_room--]; + room = &(hotel->rooms[*room_num]); + room->occupant = occupant; + + /* Assign the event and make it pending */ + if (NULL != hotel->evbase) { + event_add(&(room->eviction_timer_event), + &(hotel->eviction_timeout)); + } + + return PMIX_SUCCESS; +} + +/** + * Same as pmix_hotel_checkin(), but slightly optimized for when the + * caller *knows* that there is a room available. + */ +static inline void pmix_hotel_checkin_with_res(pmix_hotel_t *hotel, + void *occupant, + int *room_num) +{ + pmix_hotel_room_t *room; + + /* Put this occupant into the first empty room that we have */ + *room_num = hotel->unoccupied_rooms[hotel->last_unoccupied_room--]; + room = &(hotel->rooms[*room_num]); + assert(room->occupant == NULL); + room->occupant = occupant; + + /* Assign the event and make it pending */ + if (NULL != hotel->evbase) { + event_add(&(room->eviction_timer_event), + &(hotel->eviction_timeout)); + } +} + +/** + * Check the specified occupant out of the hotel. + * + * @param hotel Pointer to hotel (IN) + * @param room Room number to checkout (IN) + * + * If there is an occupant in the room, their timer is canceled and + * they are checked out. + * + * Nothing is returned (as a minor optimization). + */ +static inline void pmix_hotel_checkout(pmix_hotel_t *hotel, int room_num) +{ + pmix_hotel_room_t *room; + + /* Bozo check */ + assert(room_num < hotel->num_rooms); + + /* If there's an occupant in the room, check them out */ + room = &(hotel->rooms[room_num]); + if (PMIX_LIKELY(NULL != room->occupant)) { + /* Do not change this logic without also changing the same + logic in pmix_hotel_checkout_and_return_occupant() and + pmix_hotel.c:local_eviction_callback(). */ + room->occupant = NULL; + if (NULL != hotel->evbase) { + event_del(&(room->eviction_timer_event)); + } + hotel->last_unoccupied_room++; + assert(hotel->last_unoccupied_room < hotel->num_rooms); + hotel->unoccupied_rooms[hotel->last_unoccupied_room] = room_num; + } + + /* Don't bother returning whether we actually checked someone out + or not (because this is in the critical performance path) -- + assume the upper layer knows what it's doing. */ +} + +/** + * Check the specified occupant out of the hotel and return the occupant. + * + * @param hotel Pointer to hotel (IN) + * @param room Room number to checkout (IN) + * @param void * occupant (OUT) + * If there is an occupant in the room, their timer is canceled and + * they are checked out. + * + * Use this checkout and when caller needs the occupant + */ +static inline void pmix_hotel_checkout_and_return_occupant(pmix_hotel_t *hotel, int room_num, void **occupant) +{ + pmix_hotel_room_t *room; + + /* Bozo check */ + assert(room_num < hotel->num_rooms); + + /* If there's an occupant in the room, check them out */ + room = &(hotel->rooms[room_num]); + if (PMIX_LIKELY(NULL != room->occupant)) { + pmix_output (10, "checking out occupant %p from room num %d", room->occupant, room_num); + /* Do not change this logic without also changing the same + logic in pmix_hotel_checkout() and + pmix_hotel.c:local_eviction_callback(). */ + *occupant = room->occupant; + room->occupant = NULL; + if (NULL != hotel->evbase) { + event_del(&(room->eviction_timer_event)); + } + hotel->last_unoccupied_room++; + assert(hotel->last_unoccupied_room < hotel->num_rooms); + hotel->unoccupied_rooms[hotel->last_unoccupied_room] = room_num; + } + else { + *occupant = NULL; + } +} + +/** + * Returns true if the hotel is empty (no occupant) + * @param hotel Pointer to hotel (IN) + * @return bool true if empty false if there is a occupant(s) + * + */ +static inline bool pmix_hotel_is_empty (pmix_hotel_t *hotel) +{ + if (hotel->last_unoccupied_room == hotel->num_rooms - 1) + return true; + else + return false; +} + +/** + * Access the occupant of a room, but leave them checked into their room. + * + * @param hotel Pointer to hotel (IN) + * @param room Room number to checkout (IN) + * @param void * occupant (OUT) + * + * This accessor function is typically used to cycle across the occupants + * to check for someone already present that matches a description. + */ +static inline void pmix_hotel_knock(pmix_hotel_t *hotel, int room_num, void **occupant) +{ + pmix_hotel_room_t *room; + + /* Bozo check */ + assert(room_num < hotel->num_rooms); + + *occupant = NULL; + + /* If there's an occupant in the room, have them come to the door */ + room = &(hotel->rooms[room_num]); + if (PMIX_LIKELY(NULL != room->occupant)) { + pmix_output (10, "occupant %p in room num %d responded to knock", room->occupant, room_num); + *occupant = room->occupant; + } +} + +END_C_DECLS + +#endif /* PMIX_HOTEL_H */ diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_list.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_list.c rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.c diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_list.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_list.h rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.h diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_object.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_object.c rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.c diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_object.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_object.h rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.h diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_pointer_array.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_pointer_array.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_pointer_array.c rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_pointer_array.c diff --git a/opal/mca/pmix/pmix114/pmix/src/class/pmix_pointer_array.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_pointer_array.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/class/pmix_pointer_array.h rename to opal/mca/pmix/pmix2x/pmix/src/class/pmix_pointer_array.h diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_ring_buffer.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_ring_buffer.c new file mode 100644 index 0000000000..1967cdb185 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_ring_buffer.c @@ -0,0 +1,154 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2007 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#include +#include + +#include "pmix/pmix_common.h" +#include "src/class/pmix_ring_buffer.h" +#include "src/util/output.h" + +static void pmix_ring_buffer_construct(pmix_ring_buffer_t *); +static void pmix_ring_buffer_destruct(pmix_ring_buffer_t *); + +PMIX_CLASS_INSTANCE(pmix_ring_buffer_t, pmix_object_t, + pmix_ring_buffer_construct, + pmix_ring_buffer_destruct); + +/* + * pmix_ring_buffer constructor + */ +static void pmix_ring_buffer_construct(pmix_ring_buffer_t *ring) +{ + ring->head = 0; + ring->tail = -1; + ring->size = 0; + ring->addr = NULL; +} + +/* + * pmix_ring_buffer destructor + */ +static void pmix_ring_buffer_destruct(pmix_ring_buffer_t *ring) +{ + if( NULL != ring->addr) { + free(ring->addr); + ring->addr = NULL; + } + + ring->size = 0; +} + +/** + * initialize a ring object + */ +int pmix_ring_buffer_init(pmix_ring_buffer_t* ring, int size) +{ + /* check for errors */ + if (NULL == ring) { + return PMIX_ERR_BAD_PARAM; + } + + /* Allocate and set the ring to NULL */ + ring->addr = (char **)calloc(size * sizeof(char*), 1); + if (NULL == ring->addr) { /* out of memory */ + return PMIX_ERR_OUT_OF_RESOURCE; + } + ring->size = size; + + return PMIX_SUCCESS; +} + +void* pmix_ring_buffer_push(pmix_ring_buffer_t *ring, void *ptr) +{ + char *p=NULL; + + if (NULL != ring->addr[ring->head]) { + p = (char*)ring->addr[ring->head]; + if (ring->tail == ring->size - 1) { + ring->tail = 0; + } else { + ring->tail = ring->head + 1; + } + } + ring->addr[ring->head] = (char*)ptr; + if (ring->tail < 0) { + ring->tail = ring->head; + } + if (ring->head == ring->size - 1) { + ring->head = 0; + } else { + ring->head++; + } + return (void*)p; +} + +void* pmix_ring_buffer_pop(pmix_ring_buffer_t *ring) +{ + char *p=NULL; + + if (-1 == ring->tail) { + /* nothing has been put on the ring yet */ + p = NULL; + } else { + p = (char*)ring->addr[ring->tail]; + ring->addr[ring->tail] = NULL; + if (ring->tail == ring->size-1) { + ring->tail = 0; + } else { + ring->tail++; + } + /* see if the ring is empty */ + if (ring->tail == ring->head) { + ring->tail = -1; + } + } + return (void*)p; +} + + void* pmix_ring_buffer_poke(pmix_ring_buffer_t *ring, int i) + { + char *p=NULL; + int offset; + + if (ring->size <= i || -1 == ring->tail) { + p = NULL; + } else if (i < 0) { + /* return the value at the head of the ring */ + if (ring->head == 0) { + p = ring->addr[ring->size - 1]; + } else { + p = ring->addr[ring->head - 1]; + } + } else { + /* calculate the offset of the tail in the ring */ + offset = ring->tail + i; + /* correct for wrap-around */ + if (ring->size <= offset) { + offset -= ring->size; + } + p = ring->addr[offset]; + } + return (void*)p; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_ring_buffer.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_ring_buffer.h new file mode 100644 index 0000000000..966f917e33 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_ring_buffer.h @@ -0,0 +1,102 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2008 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** @file + * + */ + +#ifndef PMIX_RING_BUFFER_H +#define PMIX_RING_BUFFER_H + +#include + +#include "src/class/pmix_object.h" +#include "src/util/output.h" + +BEGIN_C_DECLS + +/** + * dynamic pointer ring + */ +struct pmix_ring_buffer_t { + /** base class */ + pmix_object_t super; + /* head/tail indices */ + int head; + int tail; + /** size of list, i.e. number of elements in addr */ + int size; + /** pointer to ring */ + char **addr; +}; +/** + * Convenience typedef + */ +typedef struct pmix_ring_buffer_t pmix_ring_buffer_t; +/** + * Class declaration + */ +PMIX_DECLSPEC PMIX_CLASS_DECLARATION(pmix_ring_buffer_t); + +/** + * Initialize the ring buffer, defining its size. + * + * @param ring Pointer to a ring buffer (IN/OUT) + * @param size The number of elements in the ring (IN) + * + * @return PMIX_SUCCESS if all initializations were succesful. Otherwise, + * the error indicate what went wrong in the function. + */ +PMIX_DECLSPEC int pmix_ring_buffer_init(pmix_ring_buffer_t* ring, int size); + +/** + * Push an item onto the ring buffer, displacing the oldest + * item on the ring if the ring is full + * + * @param ring Pointer to ring (IN) + * @param ptr Pointer value (IN) + * + * @return Pointer to displaced item, NULL if ring + * is not yet full + */ +PMIX_DECLSPEC void* pmix_ring_buffer_push(pmix_ring_buffer_t *ring, void *ptr); + + +/** + * Pop an item off of the ring. The oldest entry on the ring will be + * returned. If nothing on the ring, NULL is returned. + * + * @param ring Pointer to ring (IN) + * + * @return Error code. NULL indicates an error. + */ + +PMIX_DECLSPEC void* pmix_ring_buffer_pop(pmix_ring_buffer_t *ring); + +/* + * Access an element of the ring, without removing it, indexed + * starting at the tail - a value of -1 will return the element + * at the head of the ring + */ +PMIX_DECLSPEC void* pmix_ring_buffer_poke(pmix_ring_buffer_t *ring, int i); + +END_C_DECLS + +#endif /* PMIX_RING_BUFFER_H */ diff --git a/opal/mca/pmix/pmix114/pmix/src/client/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/client/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/client/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/client/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmi1.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/src/client/pmi1.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c index e37aa95c66..e0cf32377b 100644 --- a/opal/mca/pmix/pmix114/pmix/src/client/pmi1.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -66,7 +66,7 @@ PMIX_EXPORT int PMI_Init(int *spawned) pmix_info_t info[1]; bool val_optinal = 1; - if (PMIX_SUCCESS != PMIx_Init(&myproc)) { + if (PMIX_SUCCESS != PMIx_Init(&myproc, NULL, 0)) { return PMI_ERR_INIT; } @@ -121,7 +121,7 @@ PMIX_EXPORT int PMI_Finalize(void) PMI_CHECK(); pmi_init = 0; - rc = PMIx_Finalize(); + rc = PMIx_Finalize(NULL, 0); return convert_err(rc); } @@ -649,6 +649,10 @@ PMIX_EXPORT int PMI_Spawn_multiple(int count, /* push the preput values into the apps environ */ for (k = 0; k < preput_keyval_size; k++) { if (0 > asprintf(&evar, "%s=%s", preput_keyval_vector[k].key, preput_keyval_vector[k].val)) { + for (i = 0; i < count; i++) { + PMIX_APP_DESTRUCT(&apps[i]); + } + free(apps); return PMIX_ERR_NOMEM; } pmix_argv_append_nosize(&apps[i].env, evar); @@ -789,8 +793,9 @@ static int convert_err(pmix_status_t rc) return PMI_ERR_NOMEM; case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - case PMIX_ERR_COMM_FAILURE: - case PMIX_ERR_NOT_IMPLEMENTED: + case PMIX_ERR_LOST_CONNECTION_TO_SERVER: + case PMIX_ERR_LOST_PEER_CONNECTION: + case PMIX_ERR_LOST_CONNECTION_TO_CLIENT: case PMIX_ERR_NOT_SUPPORTED: case PMIX_ERR_NOT_FOUND: case PMIX_ERR_SERVER_NOT_AVAIL: diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmi2.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c similarity index 98% rename from opal/mca/pmix/pmix114/pmix/src/client/pmi2.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c index da2e8897b7..5a9ad8308e 100644 --- a/opal/mca/pmix/pmix114/pmix/src/client/pmi2.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c @@ -37,7 +37,6 @@ #include "src/include/pmix_globals.h" - #define PMI2_CHECK() \ do { \ if (!pmi2_init) { \ @@ -60,7 +59,7 @@ PMIX_EXPORT int PMI2_Init(int *spawned, int *size, int *rank, int *appnum) pmix_info_t info[1]; bool val_optinal = 1; - if (PMIX_SUCCESS != PMIx_Init(&myproc)) { + if (PMIX_SUCCESS != PMIx_Init(&myproc, NULL, 0)) { return PMI2_ERR_INIT; } @@ -146,7 +145,7 @@ PMIX_EXPORT int PMI2_Finalize(void) PMI2_CHECK(); pmi2_init = 0; - rc = PMIx_Finalize(); + rc = PMIx_Finalize(NULL, 0); return convert_err(rc); } @@ -200,6 +199,10 @@ PMIX_EXPORT int PMI2_Job_Spawn(int count, const char * cmds[], /* push the preput values into the apps environ */ for (k=0; k < preput_keyval_size; k++) { if (0 > asprintf(&evar, "%s=%s", preput_keyval_vector[j]->key, preput_keyval_vector[j]->val)) { + for (i = 0; i < count; i++) { + PMIX_APP_DESTRUCT(&apps[i]); + } + free(apps); return PMIX_ERR_NOMEM; } pmix_argv_append_nosize(&apps[i].env, evar); @@ -731,8 +734,9 @@ static int convert_err(pmix_status_t rc) return PMI2_ERR_NOMEM; case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - case PMIX_ERR_COMM_FAILURE: - case PMIX_ERR_NOT_IMPLEMENTED: + case PMIX_ERR_LOST_CONNECTION_TO_SERVER: + case PMIX_ERR_LOST_PEER_CONNECTION: + case PMIX_ERR_LOST_CONNECTION_TO_CLIENT: case PMIX_ERR_NOT_SUPPORTED: case PMIX_ERR_NOT_FOUND: case PMIX_ERR_SERVER_NOT_AVAIL: diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c similarity index 68% rename from opal/mca/pmix/pmix114/pmix/src/client/pmix_client.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c index 226b84c0ce..8736bca459 100644 --- a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c @@ -51,11 +51,12 @@ #elif PMIX_CC_USE_IDENT #ident PMIX_VERSION #endif -static const char pmix_version_string[] = PMIX_VERSION; + static const char pmix_version_string[] = PMIX_VERSION; #include "src/class/pmix_list.h" #include "src/buffer_ops/buffer_ops.h" +#include "src/event/pmix_event.h" #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/hash.h" @@ -63,92 +64,86 @@ static const char pmix_version_string[] = PMIX_VERSION; #include "src/util/progress_threads.h" #include "src/usock/usock.h" #include "src/sec/pmix_sec.h" +#include "src/include/pmix_globals.h" +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) +#include "src/dstore/pmix_dstore.h" +#endif /* PMIX_ENABLE_DSTORE */ #include "pmix_client_ops.h" #define PMIX_MAX_RETRIES 10 static pmix_status_t usock_connect(struct sockaddr *address, int *fd); -static void myerrhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) - { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client default errhandler activated"); + +static void _notify_complete(pmix_status_t status, void *cbdata) +{ + pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata; + PMIX_RELEASE(chain); } static void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { - pmix_status_t pstatus, status, rc; + pmix_status_t rc; int32_t cnt; - pmix_proc_t *procs=NULL; - size_t nprocs, ninfo; - pmix_info_t *info=NULL; pmix_cmd_t cmd; + pmix_event_chain_t *chain; + pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client_notify_recv - processing error"); - if (0 == pmix_pointer_array_get_size(&pmix_globals.errregs)) { - return; - } + "pmix:client_notify_recv - processing event"); + + /* start the local notification chain */ + chain = PMIX_NEW(pmix_event_chain_t); + chain->final_cbfunc = _notify_complete; + chain->final_cbdata = chain; + cnt=1; if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cmd, &cnt, PMIX_CMD))) { PMIX_ERROR_LOG(rc); - return; + goto error; } /* unpack the status */ cnt=1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &status, &cnt, PMIX_INT))) { - PMIX_ERROR_LOG(rc); - return; - } - pstatus = status; - - /* unpack the procs that are impacted */ - cnt=1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &nprocs, &cnt, PMIX_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &chain->status, &cnt, PMIX_INT))) { PMIX_ERROR_LOG(rc); goto error; } - if (0 < nprocs) { - PMIX_PROC_CREATE(procs, nprocs); - cnt = nprocs; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, procs, &cnt, PMIX_PROC))) { - PMIX_ERROR_LOG(rc); - goto error; - } + + /* unpack the source of the event */ + cnt=1; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &chain->source, &cnt, PMIX_PROC))) { + PMIX_ERROR_LOG(rc); + goto error; } /* unpack the info that might have been provided */ cnt=1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ninfo, &cnt, PMIX_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &chain->ninfo, &cnt, PMIX_SIZE))) { PMIX_ERROR_LOG(rc); goto error; } - if (0 < ninfo) { - PMIX_INFO_CREATE(info, ninfo); - cnt = ninfo; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, info, &cnt, PMIX_INFO))) { + if (0 < chain->ninfo) { + PMIX_INFO_CREATE(chain->info, chain->ninfo); + cnt = chain->ninfo; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, chain->info, &cnt, PMIX_INFO))) { PMIX_ERROR_LOG(rc); goto error; } } pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client_notify_recv - processed error %d, calling errhandler", pstatus); - pmix_errhandler_invoke(pstatus, procs, nprocs, info, ninfo); + "[%s:%d] pmix:client_notify_recv - processing event %d, calling errhandler", + pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status); - /* cleanup */ - PMIX_PROC_FREE(procs, nprocs); - PMIX_INFO_FREE(info, ninfo); + pmix_invoke_local_event_hdlr(chain); return; - error: + error: /* we always need to return */ pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client_notify_recv - pack error status =%d, calling def errhandler", rc); - pmix_errhandler_invoke(rc, NULL, 0, NULL, 0); - PMIX_PROC_FREE(procs, nprocs); - PMIX_INFO_FREE(info, ninfo); + "pmix:client_notify_recv - unpack error status =%d, calling def errhandler", rc); + chain = PMIX_NEW(pmix_event_chain_t); + chain->status = rc; + pmix_invoke_local_event_hdlr(chain); } @@ -178,7 +173,7 @@ static void job_data(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, /* unpack the nspace - we don't really need it, but have to * unpack it to maintain sequence */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &nspace, &cnt, PMIX_STRING))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &nspace, &cnt, PMIX_STRING))) { PMIX_ERROR_LOG(rc); return; } @@ -220,8 +215,8 @@ static pmix_status_t connect_to_server(struct sockaddr_un *address, void *cbdata /* send a request for our job info - we do this as a non-blocking * transaction because some systems cannot handle very large * blocking operations and error out if we try them. */ - req = PMIX_NEW(pmix_buffer_t); - if (PMIX_SUCCESS != (ret = pmix_bfrop.pack(req, &cmd, 1, PMIX_CMD))) { + req = PMIX_NEW(pmix_buffer_t); + if (PMIX_SUCCESS != (ret = pmix_bfrop.pack(req, &cmd, 1, PMIX_CMD))) { PMIX_ERROR_LOG(ret); PMIX_RELEASE(req); return ret; @@ -236,15 +231,15 @@ PMIX_EXPORT const char* PMIx_Get_version(void) return pmix_version_string; } -PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc) +PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, + pmix_info_t info[], size_t ninfo) { char **uri, *evar; - pmix_status_t rc; - int debug_level; + int rc, debug_level; struct sockaddr_un address; pmix_nspace_t *nsptr; pmix_cb_t cb; - int errhandler_ref; + size_t n; if (NULL == proc) { return PMIX_ERR_BAD_PARAM; @@ -254,7 +249,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc) /* since we have been called before, the nspace and * rank should be known. So return them here if * requested */ - if (NULL != proc) { + if (NULL != proc) { (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); proc->rank = pmix_globals.myid.rank; } @@ -266,6 +261,15 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc) return PMIX_ERR_INVALID_NAMESPACE; } + /* scan incoming info for directives */ + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strcmp(PMIX_EVENT_BASE, info[n].key)) { + pmix_globals.evbase = (pmix_event_base_t*)info[n].value.data.ptr; + pmix_globals.external_evbase = true; + } + } + } /* setup the globals */ pmix_globals_init(); PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t); @@ -275,8 +279,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc) /* get our effective id's */ pmix_globals.uid = geteuid(); pmix_globals.gid = getegid(); - /* default to our internal errhandler */ - pmix_add_errhandler(myerrhandler, NULL, 0, &errhandler_ref); /* initialize the output system */ if (!pmix_output_init()) { return PMIX_ERROR; @@ -363,20 +365,27 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc) pmix_globals.pindex = -1; /* setup the support */ +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + if (PMIX_SUCCESS != (rc = pmix_dstore_init())) { + return rc; + } +#endif /* PMIX_ENABLE_DSTORE */ pmix_bfrop_open(); pmix_usock_init(pmix_client_notify_recv); pmix_sec_init(); - /* create an event base and progress thread for us */ - if (NULL == (pmix_globals.evbase = pmix_start_progress_thread())) { - pmix_sec_finalize(); - pmix_usock_finalize(); - pmix_bfrop_close(); - pmix_output_close(pmix_globals.debug_output); - pmix_output_finalize(); - pmix_class_finalize(); - return -1; + if (!pmix_globals.external_evbase) { + /* create an event base and progress thread for us */ + if (NULL == (pmix_globals.evbase = pmix_start_progress_thread())) { + pmix_sec_finalize(); + pmix_usock_finalize(); + pmix_bfrop_close(); + pmix_output_close(pmix_globals.debug_output); + pmix_output_finalize(); + pmix_class_finalize(); + return -1; + } } /* setup an object to track server connection */ @@ -412,12 +421,13 @@ PMIX_EXPORT int PMIx_Initialized(void) return false; } -PMIX_EXPORT pmix_status_t PMIx_Finalize(void) +PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) { pmix_buffer_t *msg; pmix_cb_t *cb; pmix_cmd_t cmd = PMIX_FINALIZE_CMD; pmix_status_t rc; + size_t n; if (1 != pmix_globals.init_cntr) { --pmix_globals.init_cntr; @@ -429,6 +439,27 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(void) "pmix:client finalize called"); if ( 0 <= pmix_client_globals.myserver.sd ) { + /* check to see if we are supposed to execute a + * blocking fence prior to actually finalizing */ + if (NULL != info && 0 < ninfo) { + for (n=0; n < ninfo; n++) { + if (0 == strcmp(PMIX_EMBED_BARRIER, info[n].key)) { + /* did they specify a value? */ + if (PMIX_BOOL == info[n].value.type) { + if (info[n].value.data.flag) { + /* they do want the barrier */ + PMIx_Fence(NULL, 0, NULL, 0); + } + } else { + /* providing this attribute is considered + * to be "true" by default */ + PMIx_Fence(NULL, 0, NULL, 0); + } + break; + } + } + } + /* setup a cmd message to notify the PMIx * server that we are normally terminating */ msg = PMIX_NEW(pmix_buffer_t); @@ -442,29 +473,31 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(void) /* create a callback object as we need to pass it to the * recv routine so we know which callback to use when * the return message is recvd */ - cb = PMIX_NEW(pmix_cb_t); - cb->active = true; + cb = PMIX_NEW(pmix_cb_t); + cb->active = true; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client sending finalize sync to server"); + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:client sending finalize sync to server"); /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); /* wait for the ack to return */ - PMIX_WAIT_FOR_COMPLETION(cb->active); - PMIX_RELEASE(cb); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client finalize sync received"); - } + PMIX_WAIT_FOR_COMPLETION(cb->active); + PMIX_RELEASE(cb); + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:client finalize sync received"); + } - pmix_stop_progress_thread(pmix_globals.evbase); + if (!pmix_globals.external_evbase) { + pmix_stop_progress_thread(pmix_globals.evbase); + } - pmix_usock_finalize(); - PMIX_DESTRUCT(&pmix_client_globals.myserver); - PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests); + pmix_usock_finalize(); + PMIX_DESTRUCT(&pmix_client_globals.myserver); + PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests); - if (0 <= pmix_client_globals.myserver.sd) { + if (0 <= pmix_client_globals.myserver.sd) { CLOSE_THE_SOCKET(pmix_client_globals.myserver.sd); } event_base_free(pmix_globals.evbase); @@ -473,6 +506,9 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(void) #endif pmix_bfrop_close(); pmix_sec_finalize(); +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + pmix_dstore_finalize(); +#endif /* PMIX_ENABLE_DSTORE */ pmix_globals_finalize(); @@ -484,7 +520,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(void) } PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[], - pmix_proc_t procs[], size_t nprocs) + pmix_proc_t procs[], size_t nprocs) { pmix_buffer_t *bfr; pmix_cmd_t cmd = PMIX_ABORT_CMD; @@ -541,20 +577,20 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[], /* create a callback object as we need to pass it to the * recv routine so we know which callback to use when * the return message is recvd */ - cb = PMIX_NEW(pmix_cb_t); - cb->active = true; + cb = PMIX_NEW(pmix_cb_t); + cb->active = true; /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, bfr, wait_cbfunc, cb); + PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, bfr, wait_cbfunc, cb); /* wait for the release */ - PMIX_WAIT_FOR_COMPLETION(cb->active); - PMIX_RELEASE(cb); - return PMIX_SUCCESS; -} + PMIX_WAIT_FOR_COMPLETION(cb->active); + PMIX_RELEASE(cb); + return PMIX_SUCCESS; + } -static void _putfn(int sd, short args, void *cbdata) -{ + static void _putfn(int sd, short args, void *cbdata) + { pmix_cb_t *cb = (pmix_cb_t*)cbdata; pmix_status_t rc; pmix_kval_t *kv; @@ -572,13 +608,25 @@ static void _putfn(int sd, short args, void *cbdata) /* put it in our own modex hash table in case something * internal to us wants it - our nsrecord is always * first on the list */ - if (NULL == (ns = (pmix_nspace_t*)pmix_list_get_first(&pmix_globals.nspaces))) { + if (NULL == (ns = (pmix_nspace_t*)pmix_list_get_first(&pmix_globals.nspaces))) { /* shouldn't be possible */ goto done; } +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + /* TODO: It is not safe to store data on a client side + * There is a possibility to get server/client conflict. + * Do nothing here misses PMIx_Get/PMIx_Put flow (w/o PMIx_Commit) + */ + /* + if (PMIX_SUCCESS != (rc = pmix_dstore_store(ns->nspace, pmix_globals.myid.rank, kv))) { + PMIX_ERROR_LOG(rc); + } + */ +#else if (PMIX_SUCCESS != (rc = pmix_hash_store(&ns->modex, pmix_globals.myid.rank, kv))) { PMIX_ERROR_LOG(rc); } +#endif /* PMIX_ENABLE_DSTORE */ /* pack the cache that matches the scope - global scope needs * to go into both local and remote caches */ @@ -606,7 +654,7 @@ static void _putfn(int sd, short args, void *cbdata) } } - done: + done: PMIX_RELEASE(kv); // maintain accounting cb->pstatus = rc; cb->active = false; @@ -692,15 +740,15 @@ static void _commitfn(int sd, short args, void *cbdata) /* push the message into our event base to send to the server - always * send, even if we have nothing to contribute, so the server knows * that we contributed whatever we had */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msgout, NULL, NULL); + PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msgout, NULL, NULL); - done: - cb->pstatus = rc; - cb->active = false; -} + done: + cb->pstatus = rc; + cb->active = false; + } -PMIX_EXPORT pmix_status_t PMIx_Commit(void) -{ + PMIX_EXPORT pmix_status_t PMIx_Commit(void) + { pmix_cb_t *cb; pmix_status_t rc; @@ -778,7 +826,7 @@ static void _peersfn(int sd, short args, void *cbdata) pmix_argv_free(nsprocs); rc = PMIX_SUCCESS; - done: + done: cb->pstatus = rc; cb->active = false; } @@ -948,8 +996,8 @@ static pmix_status_t send_connect_ack(int sd) /* we receive a connection acknowledgement from the server, * consisting of nothing more than a status report. If success, * then we initiate authentication method */ -static pmix_status_t recv_connect_ack(int sd) -{ + static pmix_status_t recv_connect_ack(int sd) + { pmix_status_t reply; pmix_status_t rc; struct timeval tv, save; @@ -964,55 +1012,55 @@ static pmix_status_t recv_connect_ack(int sd) if (ENOPROTOOPT == errno) { sockopt = false; } else { - return PMIX_ERR_UNREACH; - } - } else { + return PMIX_ERR_UNREACH; + } + } else { /* set a timeout on the blocking recv so we don't hang */ - tv.tv_sec = 2; - tv.tv_usec = 0; - if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); - return PMIX_ERR_UNREACH; - } + tv.tv_sec = 2; + tv.tv_usec = 0; + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); + return PMIX_ERR_UNREACH; } +} /* receive the status reply */ - rc = pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(int)); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - return rc; - } +rc = pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(int)); +if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; +} /* see if they want us to do the handshake */ - if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { - if (NULL == pmix_sec.client_handshake) { - return PMIX_ERR_HANDSHAKE_FAILED; - } - if (PMIX_SUCCESS != (rc = pmix_sec.client_handshake(sd))) { - return rc; - } - } else if (PMIX_SUCCESS != reply) { - return reply; +if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { + if (NULL == pmix_sec.client_handshake) { + return PMIX_ERR_HANDSHAKE_FAILED; } - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: RECV CONNECT CONFIRMATION"); - - /* receive our index into the server's client array */ - rc = pmix_usock_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int)); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); + if (PMIX_SUCCESS != (rc = pmix_sec.client_handshake(sd))) { return rc; } - if (sockopt) { - /* return the socket to normal */ - if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { - return PMIX_ERR_UNREACH; - } - } +} else if (PMIX_SUCCESS != reply) { + return reply; +} - return PMIX_SUCCESS; +pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: RECV CONNECT CONFIRMATION"); + + /* receive our index into the server's client array */ +rc = pmix_usock_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int)); +if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; +} +if (sockopt) { + /* return the socket to normal */ + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { + return PMIX_ERR_UNREACH; + } +} + +return PMIX_SUCCESS; } void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr) @@ -1112,8 +1160,8 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr) /* the name of the node is in the key, and the value is * a comma-delimited list of procs on that node. See if we already * have this node */ - nrec = NULL; - PMIX_LIST_FOREACH(nr2, &nsptr->nodes, pmix_nrec_t) { + nrec = NULL; + PMIX_LIST_FOREACH(nr2, &nsptr->nodes, pmix_nrec_t) { if (0 == strcmp(nr2->name, kv.key)) { nrec = nr2; break; @@ -1138,13 +1186,13 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr) /* store the hostname for each proc - again, this is * data obtained via a job-level exchange, so store it * in the job-level data hash_table */ - kp2 = PMIX_NEW(pmix_kval_t); - kp2->key = strdup(PMIX_HOSTNAME); - kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); - kp2->value->type = PMIX_STRING; - kp2->value->data.string = strdup(nrec->name); - rank = strtol(procs[j], NULL, 10); - if (PMIX_SUCCESS != (rc = pmix_hash_store(&nsptr->internal, rank, kp2))) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_HOSTNAME); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_STRING; + kp2->value->data.string = strdup(nrec->name); + rank = strtol(procs[j], NULL, 10); + if (PMIX_SUCCESS != (rc = pmix_hash_store(&nsptr->internal, rank, kp2))) { PMIX_ERROR_LOG(rc); } PMIX_RELEASE(kp2); // maintain accounting @@ -1163,14 +1211,14 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr) /* maintain accounting - but note that the kptr remains * alive and stored in the hash table! So we cannot reuse * it for some other purpose */ - PMIX_RELEASE(kptr); - } - kptr = PMIX_NEW(pmix_kval_t); - cnt = 1; - } + PMIX_RELEASE(kptr); + } + kptr = PMIX_NEW(pmix_kval_t); + cnt = 1; + } /* need to release the leftover kptr */ - PMIX_RELEASE(kptr); -} + PMIX_RELEASE(kptr); + } static pmix_status_t usock_connect(struct sockaddr *addr, int *fd) { @@ -1210,7 +1258,7 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr) attempt, without even trying to establish the connection. Handle that case in a semi-rational way by trying twice before giving up */ - if (ECONNABORTED == pmix_socket_errno) { + if (ECONNABORTED == pmix_socket_errno) { pmix_output_verbose(2, pmix_globals.debug_output, "connection to server aborted by OS - retrying"); CLOSE_THE_SOCKET(sd); @@ -1260,263 +1308,3 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr) *fd = sd; return PMIX_SUCCESS; } - -static pmix_status_t pack_regevents (pmix_buffer_t *msg, pmix_cmd_t cmd, - const pmix_info_t *info, size_t ninfo) -{ - pmix_status_t rc; - - /* pack the cmd */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) { - PMIX_ERROR_LOG(rc); - return rc; - } - /* pack the number of info */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) { - PMIX_ERROR_LOG(rc); - return rc; - } - /* pack any provided info - may be NULL */ - if (NULL != info && 0 < ninfo) { - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) { - PMIX_ERROR_LOG(rc); - return rc; - } - } - - return PMIX_SUCCESS; -} - -static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata) -{ - pmix_cb_t *cb = (pmix_cb_t*)cbdata; - pmix_status_t rc; - int ret, cnt; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: regevents callback recvd"); - - if ((NULL == cb) || (NULL == cb->errreg_cbfunc)) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return; - } - /* unpack the status code */ - cnt = 1; - if ((PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ret, &cnt, PMIX_INT))) || - (PMIX_SUCCESS != ret)) { - /* This is not an actual error since the server may not support registration events */ - /* remove the err handler and call the error handler reg completion callback fn.*/ - rc = pmix_remove_errhandler(cb->errhandler_ref); - /* call the callback with error */ - cb->errreg_cbfunc(PMIX_ERR_SERVER_FAILED_REQUEST, -1, cb->cbdata); - } - else { - /* complete err handler registration with success status*/ - pmix_output_verbose(2, pmix_globals.debug_output, - "client:reg events cbfunc received status %d for errhandler %d", - ret, cb->errhandler_ref); - cb->errreg_cbfunc(PMIX_SUCCESS, cb->errhandler_ref, cb->cbdata); - } - PMIX_RELEASE(cb); -} - -void pmix_client_register_errhandler(pmix_info_t info[], size_t ninfo, - pmix_notification_fn_t errhandler, - pmix_errhandler_reg_cbfunc_t cbfunc, - void *cbdata) -{ - /* add err handler, process info keys and register for events and call the callback */ - int rc, index = 0; - pmix_buffer_t *msg; - pmix_cb_t *cb; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: register errhandler"); - /* check if this handler is already registered if so return error */ - if (PMIX_SUCCESS == pmix_lookup_errhandler (errhandler, &index)) { - /* complete request with error status and return its original reference */ - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: register errhandler - already registered"); - cbfunc(PMIX_EXISTS, index, cbdata); - - } else { - if(PMIX_SUCCESS != (rc = pmix_add_errhandler (errhandler, info, ninfo, &index))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: register errhandler - error status rc=%d", rc); - /* complete request with error*/ - cbfunc(rc, index, cbdata); - } - else { - /* To do: need to determine if the client needs to process the info keys before passing it to - server */ - pmix_output_verbose(10, pmix_globals.debug_output, - "pmix: register errhandler - added index=%d, ninfo =%lu", index, ninfo); - msg = PMIX_NEW(pmix_buffer_t); - if (PMIX_SUCCESS != (rc = pack_regevents(msg, PMIX_REGEVENTS_CMD, info, ninfo))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: register errhandler - pack events failed status=%d", rc); - PMIX_RELEASE(msg); - pmix_remove_errhandler(index); - cbfunc(PMIX_ERR_PACK_FAILURE, -1, cbdata); - } - else { - /* create a callback object as we need to pass it to the - * recv routine so we know which callback to use when - * the server acks/nacks the register events request*/ - pmix_output_verbose(10, pmix_globals.debug_output, - "pmix: register errhandler - pack events success status=%d", rc); - cb = PMIX_NEW(pmix_cb_t); - cb->errreg_cbfunc = cbfunc; - cb->cbdata = cbdata; - cb->errhandler_ref = index; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, regevents_cbfunc, cb); - } - } - } -} - -static void deregevents_cbfunc(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata) -{ - pmix_cb_t *cb = (pmix_cb_t*)cbdata; - pmix_status_t rc; - int ret, cnt =1; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: deregevents_cbfunc recvd"); - - if ((NULL == cb) || (NULL == cb->op_cbfunc)) { - return; - } - /* unpack the status code */ - if ((PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ret, &cnt, PMIX_INT))) || - (PMIX_SUCCESS != ret)) { - PMIX_ERROR_LOG(rc); - - } - /* remove the err handler and call the error handler reg completion callback fn.*/ - pmix_remove_errhandler(cb->errhandler_ref); - pmix_output_verbose(2, pmix_globals.debug_output, - "client:dereg events cbfunc received status %d for errhandler %d", - ret, cb->errhandler_ref); - cb->op_cbfunc(ret, cb->cbdata); - PMIX_RELEASE(cb); -} - -void pmix_client_deregister_errhandler(int errhandler_ref, - pmix_op_cbfunc_t cbfunc, - void *cbdata) -{ - pmix_status_t rc; - pmix_error_reg_info_t *errreg; - pmix_buffer_t *msg; - pmix_cb_t *cb; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_client_deregister_errhandler errhandler_ref = %d", errhandler_ref); - errreg = (pmix_error_reg_info_t *) pmix_pointer_array_get_item (&pmix_globals.errregs, errhandler_ref); - if (NULL != errreg ) { - msg = PMIX_NEW(pmix_buffer_t); - if (PMIX_SUCCESS != (rc = pack_regevents(msg, PMIX_DEREGEVENTS_CMD, errreg->info, errreg->ninfo))) { - PMIX_RELEASE(msg); - pmix_remove_errhandler(errhandler_ref); - cbfunc(PMIX_ERR_PACK_FAILURE, cbdata); - } - else { - /* create a callback object as we need to pass it to the - * recv routine so we know which callback to use when - * the server acks/nacks the register events request*/ - cb = PMIX_NEW(pmix_cb_t); - cb->op_cbfunc = cbfunc; - cb->cbdata = cbdata; - cb->errhandler_ref = errhandler_ref; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, deregevents_cbfunc, cb); - } - } - else - cbfunc(PMIX_ERR_NOT_FOUND, cbdata); -} - -static void notifyerror_cbfunc(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata) -{ - pmix_cb_t *cb = (pmix_cb_t*)cbdata; - pmix_status_t rc; - int ret, cnt; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: notifyerror_cbfunc recvd"); - - if ((NULL == cb) || (NULL == cb->op_cbfunc)) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return; - } - /* unpack the status code */ - if ((PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ret, &cnt, PMIX_INT))) || - (PMIX_SUCCESS != ret)) { - PMIX_ERROR_LOG(rc); - - } - /* call the notify error completion callback fn.*/ - - pmix_output_verbose(2, pmix_globals.debug_output, - "client: notified error cbfunc received status %d ", - ret); - cb->op_cbfunc(ret, cb->cbdata); - PMIX_RELEASE(cb); -} - -pmix_status_t pmix_client_notify_error(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_proc_t error_procs[], size_t error_nprocs, - pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) -{ - pmix_status_t rc; - pmix_buffer_t *msg = PMIX_NEW(pmix_buffer_t); - pmix_cmd_t cmd = PMIX_NOTIFY_CMD; - pmix_cb_t *cb; - /* pack the command */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - /* pack the status */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &status, 1, PMIX_INT))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - /* pack the error procs */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &error_nprocs, 1, PMIX_SIZE))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - if (0 < error_nprocs) { - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, error_procs, error_nprocs, PMIX_PROC))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - } - /* pack the info */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - if (0 < ninfo) { - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - } - /* create a callback object as we need to pass it to the - * recv routine so we know which callback to use when - * the server acks/nacks the register events request*/ - cb = PMIX_NEW(pmix_cb_t); - cb->op_cbfunc = cbfunc; - cb->cbdata = cbdata; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, notifyerror_cbfunc, cb); - return PMIX_SUCCESS; - cleanup: - PMIX_RELEASE(msg); - cbfunc(rc, cbdata); - return rc; -} diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_connect.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/client/pmix_client_connect.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_fence.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/client/pmix_client_fence.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_get.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c similarity index 96% rename from opal/mca/pmix/pmix114/pmix/src/client/pmix_client_get.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c index 2a1747793e..85521bdcd1 100644 --- a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_get.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -54,6 +54,9 @@ #include "src/util/progress_threads.h" #include "src/usock/usock.h" #include "src/sec/pmix_sec.h" +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) +#include "src/dstore/pmix_dstore.h" +#endif /* PMIX_ENABLE_DSTORE */ #include "pmix_client_ops.h" @@ -280,6 +283,9 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, goto done; } +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + rc = pmix_dstore_fetch(nptr->nspace, cb->rank, cb->key, &val); +#else /* we received the entire blob for this process, so * unpack and store it in the modex - this could consist * of buffers from multiple scopes */ @@ -327,8 +333,9 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, } else { rc = PMIX_SUCCESS; } +#endif /* PMIX_ENABLE_DSTORE */ - done: +done: /* if a callback was provided, execute it */ if (NULL != cb && NULL != cb->value_cbfunc) { if (NULL == val) { @@ -349,7 +356,11 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, if (0 == strncmp(nptr->nspace, cb->nspace, PMIX_MAX_NSLEN) && cb->rank == rank) { /* we have the data - see if we can find the key */ val = NULL; +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + rc = pmix_dstore_fetch(nptr->nspace, rank, cb->key, &val); +#else rc = pmix_hash_fetch(&nptr->modex, rank, cb->key, &val); +#endif /* PMIX_ENABLE_DSTORE */ cb->value_cbfunc(rc, val, cb->cbdata); if (NULL != val) { PMIX_VALUE_RELEASE(val); @@ -409,7 +420,11 @@ static void _getnbfn(int fd, short flags, void *cbdata) /* if the rank is WILDCARD, then they want all the job-level info, * so no need to check the modex */ if (PMIX_RANK_WILDCARD != cb->rank) { +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + if (PMIX_SUCCESS == (rc = pmix_dstore_fetch(nptr->nspace, cb->rank, NULL, &val))) { +#else if (PMIX_SUCCESS == (rc = pmix_hash_fetch(&nptr->modex, cb->rank, NULL, &val))) { +#endif /* PMIX_ENABLE_DSTORE */ pmix_output_verbose(2, pmix_globals.debug_output, "pmix_get[%d]: value retrieved from dstore", __LINE__); /* since we didn't provide them with a key, the hash function @@ -527,7 +542,11 @@ static void _getnbfn(int fd, short flags, void *cbdata) /* not finding it is not an error - it could be in the * modex hash table, so check it */ +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + if (PMIX_SUCCESS == (rc = pmix_dstore_fetch(nptr->nspace, cb->rank, cb->key, &val))) { +#else if (PMIX_SUCCESS == (rc = pmix_hash_fetch(&nptr->modex, cb->rank, cb->key, &val))) { +#endif /* PMIX_ENABLE_DSTORE */ pmix_output_verbose(2, pmix_globals.debug_output, "pmix_get[%d]: value retrieved from dstore", __LINE__); /* found it - we are in an event, so we can diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h new file mode 100644 index 0000000000..b2c25ab251 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_CLIENT_OPS_H +#define PMIX_CLIENT_OPS_H + +#include + + +#include "src/buffer_ops/buffer_ops.h" +#include "src/class/pmix_hash_table.h" +#include "src/usock/usock.h" + +BEGIN_C_DECLS + +typedef struct { + pmix_peer_t myserver; // messaging support to/from my server + pmix_list_t pending_requests; // list of pmix_cb_t pending data requests +} pmix_client_globals_t; + +extern pmix_client_globals_t pmix_client_globals; + +void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr); + + +END_C_DECLS + +#endif /* PMIX_CLIENT_OPS_H */ diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_pub.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/client/pmix_client_pub.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c diff --git a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_spawn.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/src/client/pmix_client_spawn.c rename to opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c index 93fa6f5eb8..73b0e85096 100644 --- a/opal/mca/pmix/pmix114/pmix/src/client/pmix_client_spawn.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c @@ -154,11 +154,11 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin return rc; } if (0 < napps) { - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, apps, napps, PMIX_APP))) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - return rc; - } + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, apps, napps, PMIX_APP))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + return rc; + } } /* create a callback object as we need to pass it to the diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/dstore/Makefile.am new file mode 100644 index 0000000000..5ad482f884 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/Makefile.am @@ -0,0 +1,16 @@ +# Copyright (c) 2015-2016 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ + + +headers += \ + src/dstore/pmix_dstore.h \ + src/dstore/pmix_esh.h + +sources += \ + src/dstore/pmix_dstore.c \ + src/dstore/pmix_esh.c diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_dstore.c b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_dstore.c new file mode 100644 index 0000000000..1cc7e7fbf5 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_dstore.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include "src/include/pmix_globals.h" + +#include "pmix_dstore.h" +#include "pmix_esh.h" + + +/* + * Array of all possible DSTOREs + */ + +/**** ENSURE THE FOLLOWING VALUE IS AT LEAST AS + **** LARGE AS THE TOTAL NUMBER OF SUPPORTED SPCs + **** IN THE ARRAY BELOW + */ + +static pmix_dstore_base_module_t *all[] = { + &pmix_dstore_esh_module, + + /* Always end the array with a NULL */ + NULL +}; + +pmix_dstore_base_module_t pmix_dstore = {0}; + +int pmix_dstore_init(void) +{ + pmix_dstore = *all[0]; + + if (!pmix_dstore.init) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_dstore.init(); +} + +void pmix_dstore_finalize(void) +{ + if (!pmix_dstore.finalize) { + pmix_dstore.finalize(); + } + + return ; +} + +int pmix_dstore_store(const char *nspace, int rank, pmix_kval_t *kv) +{ + if (!pmix_dstore.store) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_dstore.store(nspace, rank, kv); +} + +int pmix_dstore_fetch(const char *nspace, int rank, const char *key, pmix_value_t **kvs) +{ + if (!pmix_dstore.fetch) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_dstore.fetch(nspace, rank, key, kvs); +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_dstore.h b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_dstore.h new file mode 100644 index 0000000000..ee813729ce --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_dstore.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_DSTORE_H +#define PMIX_DSTORE_H + +#include + + +#include +#include "src/buffer_ops/buffer_ops.h" + + +BEGIN_C_DECLS + + +int pmix_dstore_init(void); +void pmix_dstore_finalize(void); +int pmix_dstore_store(const char *nspace, int rank, pmix_kval_t *kv); +int pmix_dstore_fetch(const char *nspace, int rank, const char *key, pmix_value_t **kvs); + +/** + * Initialize the module. Returns an error if the module cannot + * run, success if it can and wants to be used. + */ +typedef int (*pmix_dstore_base_module_init_fn_t)(void); + +/** + * Finalize the module. Tear down any allocated storage, disconnect + * from any system support. + */ +typedef int (*pmix_dstore_base_module_fini_fn_t)(void); + +/** +* store key/value pair in datastore. +* +* @param nspace namespace string +* +* @param rank rank. +* +* @param kv key/value pair. +* +* @return PMIX_SUCCESS on success. +*/ +typedef int (*pmix_dstore_base_module_store_fn_t)(const char *nspace, int rank, pmix_kval_t *kv); + +/** +* fetch value in datastore. +* +* @param nspace namespace string +* +* @param rank rank. +* +* @param key key. +* +* @return kvs(key/value pair) and PMIX_SUCCESS on success. +*/ +typedef int (*pmix_dstrore_base_module_fetch_fn_t)(const char *nspace, int rank, const char *key, pmix_value_t **kvs); + + +/** +* structure for dstore modules +*/ +typedef struct { + const char *name; + pmix_dstore_base_module_init_fn_t init; + pmix_dstore_base_module_fini_fn_t finalize; + pmix_dstore_base_module_store_fn_t store; + pmix_dstrore_base_module_fetch_fn_t fetch; +} pmix_dstore_base_module_t; + +END_C_DECLS + +#endif /* PMIX_DSTORE_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c new file mode 100644 index 0000000000..2318c6dcd7 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c @@ -0,0 +1,1447 @@ +/* + * Copyright (c) 2015-2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include + +#include +#include +#include "src/include/pmix_globals.h" + +#include "src/class/pmix_list.h" +#include "src/buffer_ops/buffer_ops.h" +#include "src/buffer_ops/types.h" +#include "src/util/pmix_environ.h" +#include "src/util/hash.h" +#include "src/util/error.h" +#include "src/sm/pmix_sm.h" + +#include "pmix_dstore.h" +#include "pmix_esh.h" + + +static int _esh_init(void); +static int _esh_finalize(void); +static int _esh_store(const char *nspace, int rank, pmix_kval_t *kv); +static int _esh_fetch(const char *nspace, int rank, const char *key, pmix_value_t **kvs); + +pmix_dstore_base_module_t pmix_dstore_esh_module = { + "esh", + _esh_init, + _esh_finalize, + _esh_store, + _esh_fetch +}; + +#define ESH_REGION_EXTENSION "EXTENSION_SLOT" +#define ESH_REGION_INVALIDATED "INVALIDATED" +#define ESH_ENV_INITIAL_SEG_SIZE "INITIAL_SEG_SIZE" +#define ESH_ENV_NS_META_SEG_SIZE "NS_META_SEG_SIZE" +#define ESH_ENV_NS_DATA_SEG_SIZE "NS_DATA_SEG_SIZE" +#define ESH_ENV_LINEAR "SM_USE_LINEAR_SEARCH" + +#define EXT_SLOT_SIZE (PMIX_MAX_KEYLEN + 1 + 2*sizeof(size_t)) /* in ext slot new offset will be stored in case if new data were added for the same process during next commit */ +#define KVAL_SIZE(size) (PMIX_MAX_KEYLEN + 1 + sizeof(size_t) + size) + +static int _store_data_for_rank(ns_track_elem_t *ns_info, int rank, pmix_buffer_t *buf); +static seg_desc_t *_create_new_segment(segment_type type, char *nsname, uint32_t id); +static seg_desc_t *_attach_new_segment(segment_type type, char *nsname, uint32_t id); +static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info); +static int _put_ns_info_to_initial_segment(const char *nspace, pmix_sm_seg_t *metaseg, pmix_sm_seg_t *dataseg); +static ns_seg_info_t *_get_ns_info_from_initial_segment(const char *nspace); +static ns_track_elem_t *_get_track_elem_for_namespace(const char *nspace); +static rank_meta_info *_get_rank_meta_info(int rank, seg_desc_t *segdesc); +static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset); +static void _update_initial_segment_info(void); +static void _set_constants_from_env(void); +static void _delete_sm_desc(seg_desc_t *desc); +static int _pmix_getpagesize(void); +static inline uint32_t _get_univ_size(const char *nspace); + +static seg_desc_t *_global_sm_seg_first; +static seg_desc_t *_global_sm_seg_last; +static pmix_list_t _namespace_info_list; +static char *_tmp_dir = NULL; +static size_t _initial_segment_size = 0; +static size_t _max_ns_num; +static size_t _meta_segment_size = 0; +static size_t _max_meta_elems; +static size_t _data_segment_size = 0; +static int _lockfd; +static char *_lockfile_name; + +/* If _direct_mode is set, it means that we use linear search + * along the array of rank meta info objects inside a meta segment + * to find the requested rank. Otherwise, we do a fast lookup + * based on rank and directly compute offset. + * This mode is called direct because it's effectively used in + * sparse communication patterns when direct modex is usually used. + */ +static int _direct_mode = 0; + +static void ncon(ns_track_elem_t *p) { + p->meta_seg = NULL; + p->data_seg = NULL; + p->num_meta_seg = 0; + p->num_data_seg = 0; +} + +static void ndes(ns_track_elem_t *p) { + _delete_sm_desc(p->meta_seg); + _delete_sm_desc(p->data_seg); +} + +PMIX_CLASS_INSTANCE(ns_track_elem_t, + pmix_list_item_t, + ncon, ndes); + +static inline int _is_server(void) +{ + return (pmix_globals.server); +} + +static inline const char *_unique_id(void) +{ + static const char *str = NULL; + if (!str) { + /* see: pmix_server.c initialize_server_base() + * to get format of uri + */ + if (_is_server()) { + static char buf[100]; + snprintf(buf, sizeof(buf) - 1, "pmix-%d", getpid()); + str = buf; + } else { + str = getenv("PMIX_SERVER_URI"); + if (str) { + str = strrchr(str, '/'); + } + str = (str ? str + 1 : "$$$"); + } + } + return str; +} + +int _esh_init(void) +{ + int rc; + seg_desc_t *seg; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s", __FILE__, __LINE__, __func__)); + + rc = pmix_sm_init(); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + PMIX_CONSTRUCT(&_namespace_info_list, pmix_list_t); + _global_sm_seg_first = NULL; + _global_sm_seg_last = NULL; + _tmp_dir = strdup(pmix_tmp_directory()); + _set_constants_from_env(); + _max_ns_num = (_initial_segment_size - sizeof(size_t) - sizeof(int)) / sizeof(ns_seg_info_t); + _max_meta_elems = (_meta_segment_size - sizeof(size_t)) / sizeof(rank_meta_info); + + /* create a lock file to prevent clients from reading while server is writing to the shared memory. + * This situation is quite often, especially in case of direct modex when clients might ask for data + * simultaneously.*/ + _lockfile_name = malloc(256); + snprintf(_lockfile_name, 256, "%s/%s_dstore_sm.lock", _tmp_dir, _unique_id()); + if (_is_server()) { + _lockfd = open(_lockfile_name, O_CREAT | O_RDWR | O_EXCL, 0600); + /* if previous launch was crashed, the lockfile might not be deleted and unlocked, + * so we delete it and create a new one. */ + if (-1 == _lockfd) { + unlink(_lockfile_name); + _lockfd = open(_lockfile_name, O_CREAT | O_RDWR, 0600); + } + } else { + _lockfd = open(_lockfile_name, O_RDWR); + } + if (-1 == _lockfd) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + + if (_is_server()) { + seg = _create_new_segment(INITIAL_SEGMENT, NULL, 0); + } else { + seg = _attach_new_segment(INITIAL_SEGMENT, NULL, 0); + } + if (NULL != seg) { + _global_sm_seg_first = seg; + _global_sm_seg_last = _global_sm_seg_first; + return PMIX_SUCCESS; + } + return PMIX_ERROR; +} + +int _esh_finalize(void) +{ + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s", __FILE__, __LINE__, __func__)); + + _delete_sm_desc(_global_sm_seg_first); + PMIX_LIST_DESTRUCT(&_namespace_info_list); + if (NULL != _tmp_dir) { + free(_tmp_dir); + } + close(_lockfd); + if (_is_server()) { + unlink(_lockfile_name); + } + free(_lockfile_name); + + pmix_sm_finalize(); + + return PMIX_SUCCESS; +} + + +int _esh_store(const char *nspace, int rank, pmix_kval_t *kv) +{ + int rc; + ns_track_elem_t *elem; + pmix_buffer_t pbkt, xfer; + ns_seg_info_t ns_info; + + if (NULL == kv) { + return PMIX_ERROR; + } + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for %s:%d", + __FILE__, __LINE__, __func__, nspace, rank)); + + /* set exclusive lock */ + flock(_lockfd, LOCK_EX); + + /* First of all, we go through local track list (list of ns_track_elem_t structures) + * and look for an element for the target namespace. + * If it is there, then shared memory segments for it are created, so we take it. + * Otherwise, create a new element, fill its fields, create corresponding meta + * and data segments for this namespace, add it to the local track list, + * and put this info (ns_seg_info_t) to the initial segment. If initial segment + * if full, then extend it by creating a new one and mark previous one as full. + * All this stuff is done inside _get_track_elem_for_namespace function. + */ + + elem = _get_track_elem_for_namespace(nspace); + if (NULL == elem) { + PMIX_ERROR_LOG(PMIX_ERROR); + /* unset lock */ + flock(_lockfd, LOCK_UN); + return PMIX_ERROR; + } + + /* If a new element was just created, we need to create corresponding meta and + * data segments and update corresponding element's fields. */ + if (NULL == elem->meta_seg || NULL == elem->data_seg) { + strncpy(ns_info.ns_name, nspace, PMIX_MAX_NSLEN+1); + ns_info.num_meta_seg = 1; + ns_info.num_data_seg = 1; + rc = _update_ns_elem(elem, &ns_info); + if (PMIX_SUCCESS != rc || NULL == elem->meta_seg || NULL == elem->data_seg) { + PMIX_ERROR_LOG(rc); + /* unset lock */ + flock(_lockfd, LOCK_UN); + return PMIX_ERROR; + } + + /* zero created shared memory segments for this namespace */ + memset(elem->meta_seg->seg_info.seg_base_addr, 0, _meta_segment_size); + memset(elem->data_seg->seg_info.seg_base_addr, 0, _data_segment_size); + + /* put ns's shared segments info to the global meta segment. */ + rc = _put_ns_info_to_initial_segment(nspace, &elem->meta_seg->seg_info, &elem->data_seg->seg_info); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + /* unset lock */ + flock(_lockfd, LOCK_UN); + return rc; + } + } + + /* Now we know info about meta segment for this namespace. If meta segment + * is not empty, then we look for data for the target rank. If they present, replace it. */ + PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); + PMIX_CONSTRUCT(&xfer, pmix_buffer_t); + PMIX_LOAD_BUFFER(&xfer, kv->value->data.bo.bytes, kv->value->data.bo.size); + pmix_buffer_t *pxfer = &xfer; + pmix_bfrop.pack(&pbkt, &pxfer, 1, PMIX_BUFFER); + xfer.base_ptr = NULL; + xfer.bytes_used = 0; + + rc = _store_data_for_rank(elem, rank, &pbkt); + + PMIX_DESTRUCT(&xfer); + PMIX_DESTRUCT(&pbkt); + + /* unset lock */ + flock(_lockfd, LOCK_UN); + return rc; +} + +int _esh_fetch(const char *nspace, int rank, const char *key, pmix_value_t **kvs) +{ + ns_seg_info_t *ns_info = NULL; + int rc; + ns_track_elem_t *elem; + rank_meta_info *rinfo = NULL; + size_t kval_cnt; + seg_desc_t *meta_seg, *data_seg; + uint8_t *addr; + pmix_buffer_t buffer; + pmix_value_t val; + uint32_t nprocs; + int cur_rank; + + if (NULL == key) { + PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output, + "dstore: Does not support passed parameters")); + return PMIX_ERROR; + } + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for %s:%d look for key %s", + __FILE__, __LINE__, __func__, nspace, rank, key)); + + if (kvs) { + *kvs = NULL; + } + + if (PMIX_RANK_UNDEF == rank) { + nprocs = _get_univ_size(nspace); + cur_rank = -1; + } else { + nprocs = 1; + cur_rank = rank; + } + + /* set shared lock */ + flock(_lockfd, LOCK_SH); + + /* First of all, we go through all initial segments and look at their field. + * If it’s 1, then generate name of next initial segment incrementing id by one and attach to it. + * We need this step to synchronize initial shared segments with our local track list. + * Then we look for the target namespace in all initial segments. + * If it is found, we get numbers of meta & data segments and + * compare these numbers with the number of trackable meta & data + * segments for this namespace in the local track list. + * If the first number exceeds the last, or the local track list + * doesn't track current namespace yet, then we update it (attach + * to additional segments). + */ + + /* first update local information about initial segments. they can be extended, so then we need to attach to new segments. */ + _update_initial_segment_info(); + + /* get information about shared segments per this namespace from the initial segment. */ + ns_info = _get_ns_info_from_initial_segment(nspace); + if (NULL == ns_info) { + /* no data for this namespace is found in the shared memory. */ + PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output, + "%s:%d:%s: no data for ns %s is found in the shared memory.", + __FILE__, __LINE__, __func__, nspace)); + /* unset lock */ + flock(_lockfd, LOCK_UN); + return PMIX_ERROR; + } + + /* get ns_track_elem_t object for the target namespace from the local track list. */ + elem = _get_track_elem_for_namespace(nspace); + if (NULL == elem) { + PMIX_ERROR_LOG(PMIX_ERROR); + /* unset lock */ + flock(_lockfd, LOCK_UN); + return PMIX_ERROR; + } + /* need to update tracker: + * attach to shared memory regions for this namespace and store its info locally + * to operate with address and detach/unlink afterwards. */ + rc = _update_ns_elem(elem, ns_info); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(PMIX_ERROR); + /* unset lock */ + flock(_lockfd, LOCK_UN); + return PMIX_ERROR; + } + + /* now we know info about meta segment for this namespace. */ + meta_seg = elem->meta_seg; + data_seg = elem->data_seg; + + while (nprocs--) { + if (PMIX_RANK_UNDEF == rank) { + cur_rank++; + } + /* Then we look for the rank meta info in the shared meta segment. */ + rinfo = _get_rank_meta_info(cur_rank, meta_seg); + if (NULL == rinfo) { + PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output, + "%s:%d:%s: no data for this rank is found in the shared memory. rank %d", + __FILE__, __LINE__, __func__, cur_rank)); + rc = PMIX_ERR_PROC_ENTRY_NOT_FOUND; + continue; + } + addr = _get_data_region_by_offset(data_seg, rinfo->offset); + if (NULL == addr) { + PMIX_ERROR_LOG(PMIX_ERROR); + rc = PMIX_ERR_PROC_ENTRY_NOT_FOUND; + continue; + } + kval_cnt = rinfo->count; + /* TODO: probably PMIX_ERR_NOT_FOUND is a better way but + * setting to one initiates wrong next logic for unknown reason */ + rc = PMIX_ERROR; + + while (0 < kval_cnt) { + /* data is stored in the following format: + * key[PMIX_MAX_KEYLEN+1] + * size_t size + * byte buffer containing pmix_value, should be loaded to pmix_buffer_t and unpacked. + * next kval pair + * ..... + * EXTENSION slot which has key = EXTENSION_SLOT and a size_t value for offset to next data address for this process. + */ + if (0 == strncmp((const char *)addr, ESH_REGION_INVALIDATED, PMIX_MAX_KEYLEN+1)) { + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %s:%d, skip %s region", + __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_REGION_INVALIDATED)); + /*skip it */ + size_t size = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1); + /* go to next item, updating address */ + addr += KVAL_SIZE(size); + } else if (0 == strncmp((const char *)addr, ESH_REGION_EXTENSION, PMIX_MAX_KEYLEN+1)) { + size_t offset = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t)); + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %s:%d, reached %s with %lu value", + __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_REGION_EXTENSION, offset)); + if (0 < offset) { + /* go to next item, updating address */ + addr = _get_data_region_by_offset(data_seg, offset); + if (NULL == addr) { + /* report problem and return */ + PMIX_ERROR_LOG(PMIX_ERROR); + rc = PMIX_ERROR; + goto done; + } + } else { + /* no more data for this rank */ + PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output, + "%s:%d:%s: no more data for this rank is found in the shared memory. rank %d key %s not found", + __FILE__, __LINE__, __func__, cur_rank, key)); + break; + } + } else if (0 == strncmp((const char *)addr, key, PMIX_MAX_KEYLEN+1)) { + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %s:%d, found target key %s", + __FILE__, __LINE__, __func__, nspace, cur_rank, key)); + /* target key is found, get value */ + size_t size = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1); + addr += PMIX_MAX_KEYLEN + 1 + sizeof(size_t); + PMIX_CONSTRUCT(&buffer, pmix_buffer_t); + PMIX_LOAD_BUFFER(&buffer, addr, size); + int cnt = 1; + /* unpack value for this key from the buffer. */ + PMIX_VALUE_CONSTRUCT(&val); + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(&buffer, &val, &cnt, PMIX_VALUE))) { + PMIX_ERROR_LOG(rc); + } else if (PMIX_SUCCESS != (rc = pmix_bfrop.copy((void**)kvs, &val, PMIX_VALUE))) { + PMIX_ERROR_LOG(rc); + } + PMIX_VALUE_DESTRUCT(&val); + buffer.base_ptr = NULL; + buffer.bytes_used = 0; + PMIX_DESTRUCT(&buffer); + goto done; + } else { + char ckey[PMIX_MAX_KEYLEN+1] = {0}; + strncpy(ckey, (const char *)addr, PMIX_MAX_KEYLEN+1); + size_t size = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1); + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %s:%d, skip key %s look for key %s", __FILE__, __LINE__, __func__, nspace, cur_rank, ckey, key)); + /* go to next item, updating address */ + addr += KVAL_SIZE(size); + kval_cnt--; + } + } + } + +done: + /* unset lock */ + flock(_lockfd, LOCK_UN); + return rc; +} + +static void _set_constants_from_env() +{ + char *str; + int page_size = _pmix_getpagesize(); + + if( NULL != (str = getenv(ESH_ENV_INITIAL_SEG_SIZE)) ) { + _initial_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > _initial_segment_size) { + _initial_segment_size = (size_t)page_size; + } + } + if (0 == _initial_segment_size) { + _initial_segment_size = INITIAL_SEG_SIZE; + } + if( NULL != (str = getenv(ESH_ENV_NS_META_SEG_SIZE)) ) { + _meta_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > _meta_segment_size) { + _meta_segment_size = (size_t)page_size; + } + } + if (0 == _meta_segment_size) { + _meta_segment_size = NS_META_SEG_SIZE; + } + if( NULL != (str = getenv(ESH_ENV_NS_DATA_SEG_SIZE)) ) { + _data_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > _data_segment_size) { + _data_segment_size = (size_t)page_size; + } + } + if (0 == _data_segment_size) { + _data_segment_size = NS_DATA_SEG_SIZE; + } + if (NULL != (str = getenv(ESH_ENV_LINEAR))) { + if (1 == strtoul(str, NULL, 10)) { + _direct_mode = 1; + } + } +} + +static void _delete_sm_desc(seg_desc_t *desc) +{ + seg_desc_t *tmp; + + /* free all global segments */ + while (NULL != desc) { + tmp = desc->next; + /* detach & unlink from current desc */ + if (desc->seg_info.seg_cpid == getpid()) { + pmix_sm_segment_unlink(&desc->seg_info); + } + pmix_sm_segment_detach(&desc->seg_info); + free(desc); + desc = tmp; + } +} + +static int _pmix_getpagesize(void) +{ +#if defined(_SC_PAGESIZE ) + return sysconf(_SC_PAGESIZE); +#elif defined(_SC_PAGE_SIZE) + return sysconf(_SC_PAGE_SIZE); +#else + return 65536; /* safer to overestimate than under */ +#endif +} + +static seg_desc_t *_create_new_segment(segment_type type, char *nsname, uint32_t id) +{ + int rc; + char file_name[PMIX_PATH_MAX]; + size_t size; + seg_desc_t *new_seg = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, type, nsname, id)); + + switch (type) { + case INITIAL_SEGMENT: + size = _initial_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/%s_initial-pmix_shared-segment-%u", _tmp_dir, _unique_id(), id); + break; + case NS_META_SEGMENT: + size = _meta_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/%s_smseg-%s-%u", _tmp_dir, _unique_id(), nsname, id); + break; + case NS_DATA_SEGMENT: + size = _data_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/%s_smdataseg-%s-%d", _tmp_dir, _unique_id(), nsname, id); + break; + default: + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t)); + if (new_seg) { + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = type; + rc = pmix_sm_segment_create(&new_seg->seg_info, file_name, size); + if (PMIX_SUCCESS == rc) { + memset(new_seg->seg_info.seg_base_addr, 0, size); + } else { + free(new_seg); + new_seg = NULL; + PMIX_ERROR_LOG(rc); + } + } + return new_seg; +} + +static seg_desc_t *_attach_new_segment(segment_type type, char *nsname, uint32_t id) +{ + int rc; + seg_desc_t *new_seg = NULL; + new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t)); + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = type; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, type, nsname, id)); + + switch (type) { + case INITIAL_SEGMENT: + new_seg->seg_info.seg_size = _initial_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/%s_initial-pmix_shared-segment-%u", _tmp_dir, _unique_id(), id); + break; + case NS_META_SEGMENT: + new_seg->seg_info.seg_size = _meta_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/%s_smseg-%s-%u", _tmp_dir, _unique_id(), nsname, id); + break; + case NS_DATA_SEGMENT: + new_seg->seg_info.seg_size = _data_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/%s_smdataseg-%s-%d", _tmp_dir, _unique_id(), nsname, id); + break; + default: + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + rc = pmix_sm_segment_attach(&new_seg->seg_info); + if (PMIX_SUCCESS != rc) { + free(new_seg); + new_seg = NULL; + PMIX_ERROR_LOG(rc); + } + return new_seg; +} + +/* This function synchronizes the content of initial shared segment and the local track list. */ +static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) +{ + seg_desc_t *seg, *tmp = NULL; + size_t i, offs; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s", + __FILE__, __LINE__, __func__)); + + tmp = ns_elem->meta_seg; + if (NULL != tmp) { + while(NULL != tmp->next) { + tmp = tmp->next; + } + } + + /* synchronize number of meta segments for the target namespace. */ + for (i = ns_elem->num_meta_seg; i < info->num_meta_seg; i++) { + if (_is_server()) { + seg = _create_new_segment(NS_META_SEGMENT, info->ns_name, i); + } else { + seg = _attach_new_segment(NS_META_SEGMENT, info->ns_name, i); + } + if (NULL == seg) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + if (NULL == tmp) { + ns_elem->meta_seg = seg; + } else { + tmp->next = seg; + } + tmp = seg; + ns_elem->num_meta_seg++; + } + + tmp = ns_elem->data_seg; + if (NULL != tmp) { + while(NULL != tmp->next) { + tmp = tmp->next; + } + } + /* synchronize number of data segments for the target namespace. */ + for (i = ns_elem->num_data_seg; i < info->num_data_seg; i++) { + if (_is_server()) { + seg = _create_new_segment(NS_DATA_SEGMENT, info->ns_name, i); + if (seg) { + offs = sizeof(size_t);//shift on offset field itself + memcpy(seg->seg_info.seg_base_addr, &offs, sizeof(size_t)); + } + } else { + seg = _attach_new_segment(NS_DATA_SEGMENT, info->ns_name, i); + } + if (NULL == seg) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + if (NULL == tmp) { + ns_elem->data_seg = seg; + } else { + tmp->next = seg; + } + tmp = seg; + ns_elem->num_data_seg++; + } + + return PMIX_SUCCESS; +} + +static seg_desc_t *extend_segment(seg_desc_t *segdesc, char *nspace) +{ + seg_desc_t *tmp, *seg; + + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s", + __FILE__, __LINE__, __func__)); + /* find last segment */ + tmp = segdesc; + while (NULL != tmp->next) { + tmp = tmp->next; + } + /* create another segment, the old one is full. */ + seg = _create_new_segment(segdesc->type, nspace, tmp->id + 1); + if (NULL == seg) { + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + + tmp->next = seg; + + return seg; +} + +static int _put_ns_info_to_initial_segment(const char *nspace, pmix_sm_seg_t *metaseg, pmix_sm_seg_t *dataseg) +{ + ns_seg_info_t elem; + size_t num_elems; + num_elems = *((size_t*)(_global_sm_seg_last->seg_info.seg_base_addr)); + seg_desc_t *last_seg = _global_sm_seg_last; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s", __FILE__, __LINE__, __func__)); + + if (_max_ns_num == num_elems) { + if (NULL == (last_seg = extend_segment(last_seg, NULL))) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + /* mark previous segment as full */ + int full = 1; + memcpy((uint8_t*)(_global_sm_seg_last->seg_info.seg_base_addr + sizeof(size_t)), &full, sizeof(int)); + _global_sm_seg_last = last_seg; + memset(_global_sm_seg_last->seg_info.seg_base_addr, 0, _initial_segment_size); + } + strncpy(elem.ns_name, nspace, PMIX_MAX_NSLEN+1); + elem.num_meta_seg = 1; + elem.num_data_seg = 1; + memcpy((uint8_t*)(_global_sm_seg_last->seg_info.seg_base_addr) + sizeof(size_t) + sizeof(int) + num_elems * sizeof(ns_seg_info_t), + &elem, sizeof(ns_seg_info_t)); + num_elems++; + memcpy((uint8_t*)(_global_sm_seg_last->seg_info.seg_base_addr), &num_elems, sizeof(size_t)); + return PMIX_SUCCESS; +} + +/* clients should sync local info with information from initial segment regularly */ +static void _update_initial_segment_info(void) +{ + seg_desc_t *tmp; + tmp = _global_sm_seg_first; + + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s", __FILE__, __LINE__, __func__)); + + /* go through all global segments */ + do { + /* check if current segment was marked as full but no more next segment is in the chain */ + if (NULL == tmp->next && 1 == *((int*)((uint8_t*)(tmp->seg_info.seg_base_addr) + sizeof(size_t)))) { + tmp->next = _attach_new_segment(INITIAL_SEGMENT, NULL, tmp->id+1); + } + tmp = tmp->next; + } + while (NULL != tmp); +} + +/* this function will be used by clients to get ns data from the initial segment and add them to the tracker list */ +static ns_seg_info_t *_get_ns_info_from_initial_segment(const char *nspace) +{ + int rc; + size_t i; + seg_desc_t *tmp; + ns_seg_info_t *elem, *cur_elem; + elem = NULL; + size_t num_elems; + + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s", __FILE__, __LINE__, __func__)); + + tmp = _global_sm_seg_first; + + rc = 1; + /* go through all global segments */ + do { + num_elems = *((size_t*)(tmp->seg_info.seg_base_addr)); + for (i = 0; i < num_elems; i++) { + cur_elem = (ns_seg_info_t*)((uint8_t*)(tmp->seg_info.seg_base_addr) + sizeof(size_t) + sizeof(int) + i * sizeof(ns_seg_info_t)); + if (0 == (rc = strncmp(cur_elem->ns_name, nspace, strlen(nspace)+1))) { + break; + } + } + if (0 == rc) { + elem = cur_elem; + break; + } + tmp = tmp->next; + } + while (NULL != tmp); + return elem; +} + +static ns_track_elem_t *_get_track_elem_for_namespace(const char *nspace) +{ + ns_track_elem_t *new_elem = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: nspace %s", + __FILE__, __LINE__, __func__, nspace)); + + /* check if this namespace is already being tracked to avoid duplicating data. */ + PMIX_LIST_FOREACH(new_elem, &_namespace_info_list, ns_track_elem_t) { + if (0 == strncmp(nspace, new_elem->ns_name, PMIX_MAX_NSLEN+1)) { + /* data for this namespace should be already stored in shared memory region. */ + /* so go and just put new data. */ + PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output, + "%s:%d:%s: found nspace %s in the track list", + __FILE__, __LINE__, __func__, nspace)); + return new_elem; + } + } + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: create new object for nspace %s", + __FILE__, __LINE__, __func__, nspace)); + /* create shared memory regions for this namespace and store its info locally + * to operate with address and detach/unlink afterwards. */ + new_elem = PMIX_NEW(ns_track_elem_t); + strncpy(new_elem->ns_name, nspace, PMIX_MAX_NSLEN+1); + + pmix_list_append(&_namespace_info_list, &new_elem->super); + + return new_elem; +} + +static rank_meta_info *_get_rank_meta_info(int rank, seg_desc_t *segdesc) +{ + size_t i; + rank_meta_info *elem = NULL; + seg_desc_t *tmp = segdesc; + size_t num_elems, rel_offset; + int id; + rank_meta_info *cur_elem; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s", + __FILE__, __LINE__, __func__)); + + if (1 == _direct_mode) { + /* do linear search to find the requested rank inside all meta segments + * for this namespace. */ + /* go through all existing meta segments for this namespace */ + do { + num_elems = *((size_t*)(tmp->seg_info.seg_base_addr)); + for (i = 0; i < num_elems; i++) { + cur_elem = (rank_meta_info*)((uint8_t*)(tmp->seg_info.seg_base_addr) + sizeof(size_t) + i * sizeof(rank_meta_info)); + if (rank == (int)cur_elem->rank) { + elem = cur_elem; + break; + } + } + tmp = tmp->next; + } + while (NULL != tmp && NULL == elem); + } else { + /* directly compute index of meta segment (id) and relative offset (rel_offset) + * inside this segment for fast lookup a rank_meta_info object for the requested rank. */ + id = rank/_max_meta_elems; + rel_offset = (rank%_max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); + /* go through all existing meta segments for this namespace. + * Stop at id number if it exists. */ + while (NULL != tmp->next && 0 != id) { + tmp = tmp->next; + id--; + } + if (0 == id) { + /* the segment is found, looking for data for the target rank. */ + elem = (rank_meta_info*)((uint8_t*)(tmp->seg_info.seg_base_addr) + rel_offset); + if ( 0 == elem->offset) { + /* offset can never be 0, it means that there is no data for this rank yet. */ + elem = NULL; + } + } + } + return elem; +} + +static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) +{ + /* it's claimed that there is still no meta info for this rank stored */ + seg_desc_t *tmp; + size_t num_elems, rel_offset; + int id, count; + rank_meta_info *cur_elem; + + if (!ns_info || !rinfo) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s: nspace %s, add rank %lu offset %lu count %lu meta info", + __FILE__, __LINE__, __func__, + ns_info->ns_name, rinfo->rank, rinfo->offset, rinfo->count)); + + tmp = ns_info->meta_seg; + if (1 == _direct_mode) { + /* get the last meta segment to put new rank_meta_info at the end. */ + while (NULL != tmp->next) { + tmp = tmp->next; + } + num_elems = *((size_t*)(tmp->seg_info.seg_base_addr)); + if (_max_meta_elems <= num_elems) { + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s: extend meta segment for nspace %s", + __FILE__, __LINE__, __func__, ns_info->ns_name)); + /* extend meta segment, so create a new one */ + tmp = extend_segment(tmp, ns_info->ns_name); + if (NULL == tmp) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + ns_info->num_meta_seg++; + memset(tmp->seg_info.seg_base_addr, 0, sizeof(rank_meta_info)); + /* update number of meta segments for namespace in initial_segment */ + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ns_info->ns_name); + if (NULL == elem) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + if (ns_info->num_meta_seg != elem->num_meta_seg) { + elem->num_meta_seg = ns_info->num_meta_seg; + } + num_elems = 0; + } + cur_elem = (rank_meta_info*)((uint8_t*)(tmp->seg_info.seg_base_addr) + sizeof(size_t) + num_elems * sizeof(rank_meta_info)); + memcpy(cur_elem, rinfo, sizeof(rank_meta_info)); + num_elems++; + memcpy(tmp->seg_info.seg_base_addr, &num_elems, sizeof(size_t)); + } else { + /* directly compute index of meta segment (id) and relative offset (rel_offset) + * inside this segment for fast lookup a rank_meta_info object for the requested rank. */ + id = rinfo->rank/_max_meta_elems; + rel_offset = (rinfo->rank % _max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); + count = id; + /* go through all existing meta segments for this namespace. + * Stop at id number if it exists. */ + while (NULL != tmp->next && 0 != count) { + tmp = tmp->next; + count--; + } + /* if there is no segment with this id, then create all missing segments till the id number. */ + if ((int)ns_info->num_meta_seg < (id+1)) { + while ((int)ns_info->num_meta_seg != (id+1)) { + /* extend meta segment, so create a new one */ + tmp = extend_segment(tmp, ns_info->ns_name); + if (NULL == tmp) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + memset(tmp->seg_info.seg_base_addr, 0, sizeof(rank_meta_info)); + ns_info->num_meta_seg++; + } + /* update number of meta segments for namespace in initial_segment */ + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ns_info->ns_name); + if (NULL == elem) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + if (ns_info->num_meta_seg != elem->num_meta_seg) { + elem->num_meta_seg = ns_info->num_meta_seg; + } + } + /* store rank_meta_info object by rel_offset. */ + cur_elem = (rank_meta_info*)((uint8_t*)(tmp->seg_info.seg_base_addr) + rel_offset); + memcpy(cur_elem, rinfo, sizeof(rank_meta_info)); + } + return PMIX_SUCCESS; +} + +static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) +{ + seg_desc_t *tmp = segdesc; + size_t rel_offset = offset; + uint8_t *dataaddr = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s", + __FILE__, __LINE__, __func__)); + + /* go through all existing data segments for this namespace */ + do { + if (rel_offset >= _data_segment_size) { + rel_offset -= _data_segment_size; + } else { + dataaddr = tmp->seg_info.seg_base_addr + rel_offset; + } + tmp = tmp->next; + } + while (NULL != tmp && NULL == dataaddr); + return dataaddr; +} + +static size_t get_free_offset(seg_desc_t *data_seg) +{ + size_t offset; + seg_desc_t *tmp; + int id = 0; + tmp = data_seg; + /* first find the last data segment */ + while (NULL != tmp->next) { + tmp = tmp->next; + id++; + } + offset = *((size_t*)(tmp->seg_info.seg_base_addr)); + if (0 == offset) { + /* this is the first created data segment, the first 8 bytes are used to place the free offset value itself */ + offset = sizeof(size_t); + } + return (id * _data_segment_size + offset); +} + +static int put_empty_ext_slot(seg_desc_t *dataseg) +{ + size_t global_offset, rel_offset, data_ended, sz, val; + uint8_t *addr; + global_offset = get_free_offset(dataseg); + rel_offset = global_offset % _data_segment_size; + if (rel_offset + EXT_SLOT_SIZE > _data_segment_size) { + return PMIX_ERROR; + } + addr = _get_data_region_by_offset(dataseg, global_offset); + strncpy((char *)addr, ESH_REGION_EXTENSION, PMIX_MAX_KEYLEN+1); + val = 0; + sz = sizeof(size_t); + memcpy(addr + PMIX_MAX_KEYLEN + 1, &sz, sz); + memcpy(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t), &val, sz); + + /* update offset at the beginning of current segment */ + data_ended = rel_offset + EXT_SLOT_SIZE; + addr = (uint8_t*)(addr - rel_offset); + memcpy(addr, &data_ended, sizeof(size_t)); + return PMIX_SUCCESS; +} + +static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, char *key, void *buffer, size_t size) +{ + size_t offset; + seg_desc_t *tmp; + int id = 0; + size_t global_offset, data_ended; + uint8_t *addr; + size_t sz; + + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s: key %s", + __FILE__, __LINE__, __func__, key)); + + tmp = dataseg; + while (NULL != tmp->next) { + tmp = tmp->next; + id++; + } + global_offset = get_free_offset(dataseg); + offset = global_offset % _data_segment_size; + + /* We should provide additional space at the end of segment to place EXTENSION_SLOT to have an ability to enlarge data for this rank.*/ + if (sizeof(size_t) + KVAL_SIZE(size) + EXT_SLOT_SIZE > _data_segment_size) { + /* this is an error case: segment is so small that cannot place evem a single key-value pair. + * warn a user about it and fail. */ + offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ + pmix_output(0, "PLEASE set NS_DATA_SEG_SIZE to value which is larger when %lu.", + sizeof(size_t) + PMIX_MAX_KEYLEN + 1 + sizeof(size_t) + size + EXT_SLOT_SIZE); + return offset; + } + if (offset + KVAL_SIZE(size) + EXT_SLOT_SIZE > _data_segment_size) { + id++; + /* create a new data segment. */ + tmp = extend_segment(tmp, ns_info->ns_name); + if (NULL == tmp) { + PMIX_ERROR_LOG(PMIX_ERROR); + offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ + return offset; + } + ns_info->num_data_seg++; + /* update_ns_info_in_initial_segment */ + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ns_info->ns_name); + if (NULL == elem) { + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + elem->num_data_seg++; + + offset = sizeof(size_t); + } + global_offset = offset + id * _data_segment_size; + addr = (uint8_t*)(tmp->seg_info.seg_base_addr)+offset; + strncpy((char *)addr, key, PMIX_MAX_KEYLEN+1); + sz = size; + memcpy(addr + PMIX_MAX_KEYLEN + 1, &sz, sizeof(size_t)); + memcpy(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t), buffer, size); + + /* update offset at the beginning of current segment */ + data_ended = offset + KVAL_SIZE(size); + addr = (uint8_t*)(tmp->seg_info.seg_base_addr); + memcpy(addr, &data_ended, sizeof(size_t)); + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s: key %s, rel start offset %lu, rel end offset %lu, abs shift %lu size %lu", + __FILE__, __LINE__, __func__, key, offset, data_ended, id * _data_segment_size, size)); + return global_offset; +} + +static int pmix_sm_store(ns_track_elem_t *ns_info, int rank, pmix_kval_t *kval, rank_meta_info **rinfo, int data_exist) +{ + size_t offset, size, kval_cnt; + pmix_buffer_t *buffer; + int rc; + seg_desc_t *datadesc; + uint8_t *addr; + + PMIX_OUTPUT_VERBOSE((2, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d", + __FILE__, __LINE__, __func__, rank, data_exist)); + + datadesc = ns_info->data_seg; + /* pack value to the buffer */ + buffer = PMIX_NEW(pmix_buffer_t); + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(buffer, kval->value, 1, PMIX_VALUE))) { + PMIX_RELEASE(buffer); + PMIX_ERROR_LOG(rc); + return rc; + } + size = buffer->bytes_used; + + if (0 == data_exist) { + /* there is no data blob for this rank yet, so add it. */ + size_t free_offset; + free_offset = get_free_offset(datadesc); + offset = put_data_to_the_end(ns_info, datadesc, kval->key, buffer->base_ptr, size); + if (0 == offset) { + /* this is an error */ + PMIX_RELEASE(buffer); + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + /* if it's the first time when we put data for this rank, then *rinfo == NULL, + * and even if segment was extended, and data was put into the next segment, + * we don't need to extension slot at the end of previous segment. + * If we try, we might overwrite other segments memory, + * because previous segment is already full. */ + if (free_offset != offset && NULL != *rinfo) { + /* here we compare previous free offset with the offset where we just put data. + * It should be equal in the normal case. It it's not true, then it means that + * segment was extended, and we put data to the next segment, so we now need to + * put extension slot at the end of previous segment with a "reference" to a new_offset */ + size_t sz = sizeof(size_t); + addr = _get_data_region_by_offset(datadesc, free_offset); + strncpy((char *)addr, ESH_REGION_EXTENSION, PMIX_MAX_KEYLEN+1); + memcpy(addr + PMIX_MAX_KEYLEN + 1, &sz, sizeof(size_t)); + memcpy(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t), &offset, sizeof(size_t)); + } + if (NULL == *rinfo) { + *rinfo = (rank_meta_info*)malloc(sizeof(rank_meta_info)); + (*rinfo)->rank = rank; + (*rinfo)->offset = offset; + (*rinfo)->count = 0; + } + (*rinfo)->count++; + } else if (NULL != *rinfo) { + /* there is data blob for this rank */ + addr = _get_data_region_by_offset(datadesc, (*rinfo)->offset); + if (NULL == addr) { + PMIX_RELEASE(buffer); + PMIX_ERROR_LOG(PMIX_ERROR); + return rc; + } + /* go through previous data region and find key matches. + * If one is found, then mark this kval as invalidated. + * Then put a new empty offset to the next extension slot, + * and add new kval by this offset. + * no need to update meta info, it's still the same. */ + kval_cnt = (*rinfo)->count; + int add_to_the_end = 1; + while (0 < kval_cnt) { + /* data is stored in the following format: + * key[PMIX_MAX_KEYLEN+1] + * size_t size + * byte buffer containing pmix_value, should be loaded to pmix_buffer_t and unpacked. + * next kval pair + * ..... + * extension slot which has key = EXTENSION_SLOT and a size_t value for offset to next data address for this process. + */ + if (0 == strncmp((const char *)addr, ESH_REGION_EXTENSION, PMIX_MAX_KEYLEN+1)) { + offset = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t)); + if (0 < offset) { + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d %s is filled with %lu value", + __FILE__, __LINE__, __func__, rank, data_exist, ESH_REGION_EXTENSION, offset)); + /* go to next item, updating address */ + addr = _get_data_region_by_offset(datadesc, offset); + if (NULL == addr) { + PMIX_RELEASE(buffer); + PMIX_ERROR_LOG(PMIX_ERROR); + return rc; + } + } else { + /* should not be, we should be out of cycle when this happens */ + } + } else if (0 == strncmp((const char *)addr, kval->key, PMIX_MAX_KEYLEN+1)) { + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d found target key %s", + __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); + /* target key is found, compare value sizes */ + size_t cur_size = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1); + if (cur_size != size) { + //if (1) { /* if we want to test replacing values for existing keys. */ + /* invalidate current value and store another one at the end of data region. */ + strncpy((char *)addr, ESH_REGION_INVALIDATED, PMIX_MAX_KEYLEN+1); + /* decrementing count, it will be incremented back when we add a new value for this key at the end of region. */ + (*rinfo)->count--; + kval_cnt--; + /* go to next item, updating address */ + addr += KVAL_SIZE(cur_size); + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d mark key %s regions as invalidated. put new data at the end.", + __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); + } else { + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d replace data for key %s type %d in place", + __FILE__, __LINE__, __func__, rank, data_exist, kval->key, kval->value->type)); + /* replace old data with new one. */ + addr += PMIX_MAX_KEYLEN + 1; + memcpy(addr, &size, sizeof(size_t)); + addr += sizeof(size_t); + memset(addr, 0, cur_size); + memcpy(addr, buffer->base_ptr, size); + addr += cur_size; + add_to_the_end = 0; + break; + } + } else { + char ckey[PMIX_MAX_KEYLEN+1] = {0}; + strncpy(ckey, (const char *)addr, PMIX_MAX_KEYLEN+1); + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d skip %s key, look for %s key", + __FILE__, __LINE__, __func__, rank, data_exist, ckey, kval->key)); + /* Skip it: key is "INVALIDATED" or key is valid but different from target one. */ + if (0 != strncmp(ESH_REGION_INVALIDATED, ckey, PMIX_MAX_KEYLEN+1)) { + /* count only valid items */ + kval_cnt--; + } + size_t size = *(size_t *)(addr + PMIX_MAX_KEYLEN + 1); + /* go to next item, updating address */ + addr += KVAL_SIZE(size); + } + } + if (1 == add_to_the_end) { + /* if we get here, it means that we want to add a new item for the target rank, or + * we mark existing item with the same key as "invalidated" and want to add new item + * for the same key. */ + (*rinfo)->count++; + size_t free_offset; + free_offset = get_free_offset(datadesc); + /* add to the end */ + offset = put_data_to_the_end(ns_info, datadesc, kval->key, buffer->base_ptr, size); + if (0 == offset) { + PMIX_RELEASE(buffer); + PMIX_ERROR_LOG(PMIX_ERROR); + return PMIX_ERROR; + } + /* we just reached the end of data for the target rank, and there can be two cases: + * (1) - we are in the middle of data segment; data for this rank is separated from + * data for different ranks, and that's why next element is EXTENSION_SLOT. + * We put new data to the end of data region and just update EXTENSION_SLOT value by new offset. + */ + if (0 == strncmp((const char *)addr, ESH_REGION_EXTENSION, PMIX_MAX_KEYLEN+1)) { + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d %s should be filled with offset %lu value", + __FILE__, __LINE__, __func__, rank, data_exist, ESH_REGION_EXTENSION, offset)); + memcpy(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t), &offset, sizeof(size_t)); + } else { + /* (2) - we point to the first free offset, no more data is stored further in this segment. + * There is no EXTENSION_SLOT by this addr since we continue pushing data for the same rank, + * and there is no need to split it. + * But it's possible that we reached the end of current data region and just jumped to the new region + * to put new data, in that case free_offset != offset and we must put EXTENSION_SLOT by the current addr + * forcibly and store new offset in its value. */ + if (free_offset != offset) { + /* segment was extended, need to put extension slot by free_offset indicating new_offset */ + size_t sz = sizeof(size_t); + strncpy((char *)addr, ESH_REGION_EXTENSION, PMIX_MAX_KEYLEN+1); + memcpy(addr + PMIX_MAX_KEYLEN + 1, &sz, sz); + memcpy(addr + PMIX_MAX_KEYLEN + 1 + sizeof(size_t), &offset, sz); + } + } + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d, replace flag %d item not found ext slot empty, put key %s to the end", + __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); + } + } + buffer->base_ptr = NULL; + buffer->bytes_used = 0; + PMIX_RELEASE(buffer); + return rc; +} + +static int _store_data_for_rank(ns_track_elem_t *ns_info, int rank, pmix_buffer_t *buf) +{ + int rc; + int32_t cnt; + + pmix_buffer_t *bptr; + pmix_kval_t *kp; + seg_desc_t *metadesc, *datadesc; + + rank_meta_info *rinfo = NULL; + size_t num_elems, free_offset, new_free_offset; + int data_exist; + + PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output, + "%s:%d:%s: for rank %d", __FILE__, __LINE__, __func__, rank)); + + metadesc = ns_info->meta_seg; + datadesc = ns_info->data_seg; + + if (NULL == datadesc || NULL == metadesc) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERROR; + } + + num_elems = *((size_t*)(metadesc->seg_info.seg_base_addr)); + data_exist = 0; + /* when we don't use linear search (_direct_mode ==0 ) we don't use num_elems field, + * so anyway try to get rank_meta_info first. */ + if (0 < num_elems || 0 == _direct_mode) { + /* go through all elements in meta segment and look for target rank. */ + rinfo = _get_rank_meta_info(rank, metadesc); + if (NULL != rinfo) { + data_exist = 1; + } + } + /* incoming buffer may contain several inner buffers for different scopes, + * so unpack these buffers, and then unpack kvals from each modex buffer, + * storing them in the shared memory dstore. + */ + cnt = 1; + free_offset = get_free_offset(datadesc); + while (PMIX_SUCCESS == (rc = pmix_bfrop.unpack(buf, &bptr, &cnt, PMIX_BUFFER))) { + cnt = 1; + kp = PMIX_NEW(pmix_kval_t); + while (PMIX_SUCCESS == (rc = pmix_bfrop.unpack(bptr, kp, &cnt, PMIX_KVAL))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: unpacked key %s", kp->key); + if (PMIX_SUCCESS != (rc = pmix_sm_store(ns_info, rank, kp, &rinfo, data_exist))) { + PMIX_ERROR_LOG(rc); + } + PMIX_RELEASE(kp); // maintain acctg - hash_store does a retain + cnt = 1; + kp = PMIX_NEW(pmix_kval_t); + } + cnt = 1; + PMIX_RELEASE(kp); + PMIX_RELEASE(bptr); // free's the data region + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { + PMIX_ERROR_LOG(rc); + break; + } + } + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { + PMIX_ERROR_LOG(rc); + } else { + rc = PMIX_SUCCESS; + } + + /* Check if new data was put at the end of data segment. + * It's possible that old data just was replaced with new one, + * in that case we don't reserve space for EXTENSION_SLOT, it's + * already reserved. + * */ + new_free_offset = get_free_offset(datadesc); + if (new_free_offset != free_offset) { + /* Reserve space for EXTENSION_SLOT at the end of data blob. + * We need it to split data for one rank from data for different + * ranks and to allow extending data further. + * We also put EXTENSION_SLOT at the end of each data segment, and + * its value points to the beginning of next data segment. + * */ + rc = put_empty_ext_slot(ns_info->data_seg); + if (PMIX_SUCCESS != rc) { + if (NULL != rinfo) { + free(rinfo); + } + PMIX_ERROR_LOG(rc); + return rc; + } + } + + /* if this is the first data posted for this rank, then + * update meta info for it */ + if (0 == data_exist) { + set_rank_meta_info(ns_info, rinfo); + if (NULL != rinfo) { + free(rinfo); + } + } + + return rc; +} + +static inline uint32_t _get_univ_size(const char *nspace) +{ + pmix_value_t *val = NULL; + uint32_t nprocs = 0; + pmix_nspace_t *ns, *nptr; + + nptr = NULL; + PMIX_LIST_FOREACH(ns, &pmix_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(nspace, ns->nspace)) { + nptr = ns; + break; + } + } + + if (nptr && (PMIX_SUCCESS == pmix_hash_fetch(&nptr->internal, PMIX_RANK_WILDCARD, PMIX_UNIV_SIZE, &val))) { + if (val->type == PMIX_UINT32) { + nprocs = val->data.uint32; + } + PMIX_VALUE_RELEASE(val); + } + + return nprocs; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h new file mode 100644 index 0000000000..d47719ef51 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015-2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_DSTORE_ESH_H +#define PMIX_DSTORE_ESH_H + +#include + + +#include "pmix_dstore.h" +#include "src/sm/pmix_sm.h" + +BEGIN_C_DECLS + +#define INITIAL_SEG_SIZE 4096 +#define NS_META_SEG_SIZE (1<<22) +#define NS_DATA_SEG_SIZE (1<<22) + +typedef enum { + INITIAL_SEGMENT, + NS_META_SEGMENT, + NS_DATA_SEGMENT +} segment_type; + +/* initial segment format: + * size_t num_elems; + * int full; //indicate to client that it needs to attach to the next segment + * ns_seg_info_t ns_seg_info[max_ns_num]; + */ + +typedef struct { + char ns_name[PMIX_MAX_NSLEN+1]; + size_t num_meta_seg;/* read by clients to attach to this number of segments. */ + size_t num_data_seg; +} ns_seg_info_t; + +/* meta segment format: + * size_t num_elems; + * rank_meta_info meta_info[max_meta_elems]; + */ + +typedef struct { + size_t rank; + size_t offset; + size_t count; +} rank_meta_info; + +/* this structs are used to store information about + * shared segments addresses locally at each process, + * so they are common for different types of segments + * and don't have a specific content (namespace's info, + * rank's meta info, ranks's data). */ + +typedef struct seg_desc_t seg_desc_t; +struct seg_desc_t { + segment_type type; + pmix_sm_seg_t seg_info; + uint32_t id; + seg_desc_t *next; +}; + +typedef struct { + pmix_list_item_t super; + char ns_name[PMIX_MAX_NSLEN+1]; + size_t num_meta_seg; + size_t num_data_seg; + seg_desc_t *meta_seg; + seg_desc_t *data_seg; +} ns_track_elem_t; +PMIX_CLASS_DECLARATION(ns_track_elem_t); + +extern pmix_dstore_base_module_t pmix_dstore_esh_module; + +END_C_DECLS + +#endif /* PMIX_DSTORE_ESH_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/event/Makefile.am new file mode 100644 index 0000000000..079b1c0c11 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/event/Makefile.am @@ -0,0 +1,15 @@ +# +# Copyright (c) 2015-2016 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers += \ + src/event/pmix_event.h + +sources += \ + src/event/pmix_event_registration.c \ + src/event/pmix_event_notification.c diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h new file mode 100644 index 0000000000..95d2a3f94b --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h @@ -0,0 +1,130 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_EVENT_H +#define PMIX_EVENT_H + +#include +#include + +#include +#include "src/class/pmix_list.h" +#include "src/util/output.h" + + BEGIN_C_DECLS + +/* define an object for tracking event handlers focused on a + * single status code */ +typedef struct { + pmix_list_item_t super; + char *name; + size_t index; + pmix_status_t code; + pmix_notification_fn_t evhdlr; +} pmix_single_event_t; +PMIX_CLASS_DECLARATION(pmix_single_event_t); + +/* define an object for tracking event handlers registered + * on multiple status codes, generally corresponding to a + * functional group */ +typedef struct { + pmix_list_item_t super; + char *name; + size_t index; + pmix_status_t *codes; + size_t ncodes; + pmix_notification_fn_t evhdlr; +} pmix_multi_event_t; +PMIX_CLASS_DECLARATION(pmix_multi_event_t); + +/* define an object for tracking default event handlers */ +typedef struct { + pmix_list_item_t super; + char *name; + size_t index; + pmix_notification_fn_t evhdlr; +} pmix_default_event_t; +PMIX_CLASS_DECLARATION(pmix_default_event_t); + +/* define an object for tracking status codes we are actively + * registered to receive */ +typedef struct { + pmix_list_item_t super; + pmix_status_t code; +} pmix_active_code_t; +PMIX_CLASS_DECLARATION(pmix_active_code_t); + +/* define an object for housing the different lists of events + * we have registered so we can easily scan them in precedent + * order when we get an event */ +typedef struct { + pmix_object_t super; + size_t nhdlrs; + pmix_list_t actives; + pmix_list_t single_events; + pmix_list_t multi_events; + pmix_list_t default_events; +} pmix_events_t; +PMIX_CLASS_DECLARATION(pmix_events_t); + +/* define an object for chaining event notifications thru + * the local state machine. Each registered event handler + * that asked to be notified for a given code is given a + * chance to "see" the reported event, starting with + * single-code handlers, then multi-code handlers, and + * finally default handlers. This object provides a + * means for us to relay the event across that chain + */ +typedef struct pmix_event_chain_t { + pmix_object_t super; + pmix_status_t status; + bool nondefault; + pmix_proc_t source; + pmix_data_range_t range; + pmix_info_t *info; + size_t ninfo; + pmix_info_t *results; + size_t nresults; + pmix_single_event_t *sing; + pmix_multi_event_t *multi; + pmix_default_event_t *def; + pmix_op_cbfunc_t final_cbfunc; + void *final_cbdata; +} pmix_event_chain_t; +PMIX_CLASS_DECLARATION(pmix_event_chain_t); + +/* invoke the error handler that is registered against the given + * status, passing it the provided info on the procs that were + * affected, plus any additional info provided by the server */ + void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain); + +#define PMIX_REPORT_EVENT(e) \ + do { \ + pmix_event_chain_t *_ch; \ + _ch = PMIX_NEW(pmix_event_chain_t); \ + _ch->status = (e); \ + pmix_invoke_local_event_hdlr(_ch); \ + PMIX_RELEASE(_ch); \ + } while(0) + + + END_C_DECLS + +#endif /* PMIX_EVENT_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c new file mode 100644 index 0000000000..9180221158 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c @@ -0,0 +1,638 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include +#include + +#include +#include +#include + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "src/client/pmix_client_ops.h" +#include "src/server/pmix_server_ops.h" +#include "src/include/pmix_globals.h" + +static pmix_status_t notify_server_of_event(pmix_status_t status, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +static pmix_status_t notify_client_of_event(pmix_status_t status, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/* if we are a client, we call this function to notify the server of + * an event. If we are a server, our host RM will call this function + * to notify us of an event */ +PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + + if (pmix_globals.server) { + rc = notify_client_of_event(status, source, range, + info, ninfo, + cbfunc, cbdata); + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_server_notify_event source = %s:%d event_status = %d, rc= %d", + source->nspace, source->rank, status, rc); + } else { + rc = notify_server_of_event(status, source, range, + info, ninfo, + cbfunc, cbdata); + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_client_notify_event source = %s:%d event_status =%d, rc=%d", + source->nspace, source->rank, status, rc); + } + return rc; +} + +static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata) +{ + pmix_status_t rc, ret; + int32_t cnt = 1; + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + + /* unpack the status */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ret, &cnt, PMIX_STATUS))) { + PMIX_ERROR_LOG(rc); + ret = rc; + } + /* do the cback */ + if (NULL != cb->op_cbfunc) { + cb->op_cbfunc(ret, cb->cbdata); + } + PMIX_RELEASE(cb); +} + +/* as a client, we pass the notification to our server */ +static pmix_status_t notify_server_of_event(pmix_status_t status, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t rc; + pmix_buffer_t *msg; + pmix_cmd_t cmd = PMIX_NOTIFY_CMD; + pmix_cb_t *cb; + pmix_event_chain_t *chain; + + if (!pmix_globals.connected) { + return PMIX_ERR_UNREACH; + } + + /* create the msg object */ + msg = PMIX_NEW(pmix_buffer_t); + + /* pack the command */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + /* pack the status */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &status, 1, PMIX_STATUS))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + /* no need to pack the source as it is us */ + + /* pack the range */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &range, 1, PMIX_DATA_RANGE))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + /* pack the info */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + if (0 < ninfo) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + } + + /* setup for our own local callbacks */ + chain = PMIX_NEW(pmix_event_chain_t); + chain->status = status; + (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + chain->source.rank = pmix_globals.myid.rank; + if (0 < ninfo) { + + } + + /* create a callback object as we need to pass it to the + * recv routine so we know which callback to use when + * the server acks/nacks the register events request*/ + cb = PMIX_NEW(pmix_cb_t); + cb->op_cbfunc = cbfunc; + cb->cbdata = cbdata; + /* push the message into our event base to send to the server */ + PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, notify_event_cbfunc, cb); + + /* now notify any matching registered callbacks we have */ + pmix_invoke_local_event_hdlr(chain); + PMIX_RELEASE(chain); // maintain accounting + + return PMIX_SUCCESS; + + cleanup: + PMIX_RELEASE(msg); + /* we were unable to send anything, so we just return the error */ + return rc; +} + + +static void progress_local_event_hdlr(pmix_status_t status, + pmix_info_t *results, size_t nresults, + pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata) +{ + pmix_event_chain_t *chain = (pmix_event_chain_t*)notification_cbdata; + size_t n, nsave; + pmix_info_t *newinfo; + pmix_list_item_t *nxt; + pmix_single_event_t *sing; + pmix_multi_event_t *multi; + pmix_default_event_t *def; + + /* if the caller indicates that the chain is completed, then stop here */ + if (PMIX_EVENT_ACTION_COMPLETE == status) { + goto complete; + } + + /* save the current number of results */ + nsave = chain->nresults; + /* create the new space */ + PMIX_INFO_CREATE(newinfo, chain->nresults + nresults + 1); + /* transfer over the prior data */ + for (n=0; n < chain->nresults; n++) { + PMIX_INFO_XFER(&newinfo[n], &chain->results[n]); + } + /* save this handler's response */ + if (NULL != chain->sing) { + if (NULL != chain->sing->name) { + (void)strncpy(newinfo[nsave].key, chain->sing->name, PMIX_MAX_KEYLEN); + } + } else if (NULL != chain->multi) { + if (NULL != chain->multi->name) { + (void)strncpy(newinfo[nsave].key, chain->multi->name, PMIX_MAX_KEYLEN); + } + } else if (NULL != chain->def) { + if (NULL != chain->def->name) { + (void)strncpy(newinfo[nsave].key, chain->def->name, PMIX_MAX_KEYLEN); + } + } else { + (void)strncpy(newinfo[nsave].key, "UNKNOWN", PMIX_MAX_KEYLEN); + } + newinfo[nsave].value.type = PMIX_STATUS; + newinfo[nsave].value.data.status = status; + /* transfer across the new results */ + for (n=0; n < nresults; n++) { + PMIX_INFO_XFER(&newinfo[n+nsave+1], &results[n]); + } + /* release the prior results */ + if (0 < chain->nresults) { + PMIX_INFO_FREE(chain->results, chain->nresults); + } + /* pass along the new ones */ + chain->results = newinfo; + chain->nresults = nsave + nresults; + + /* see if we need to continue, starting with the single code events */ + if (NULL != chain->sing) { + /* the last handler was for a single code - see if there are + * any others that match this event */ + while (pmix_list_get_end(&pmix_globals.events.single_events) != (nxt = pmix_list_get_next(&chain->sing->super))) { + sing = (pmix_single_event_t*)nxt; + if (sing->code == chain->status) { + PMIX_RETAIN(chain); + chain->sing = sing; + sing->evhdlr(sing->index, + chain->status, &chain->source, + chain->info, chain->ninfo, + chain->results, chain->nresults, + progress_local_event_hdlr, (void*)chain); + goto complete; + } + } + /* if we get here, then there are no more single code + * events that match */ + chain->sing = NULL; + /* pickup the beginning of the multi-code event list */ + chain->multi = (pmix_multi_event_t*)pmix_list_get_begin(&pmix_globals.events.multi_events); + } + + /* see if we need to continue with the multi code events */ + if (NULL != chain->multi) { + while (pmix_list_get_end(&pmix_globals.events.multi_events) != (nxt = pmix_list_get_next(&chain->multi->super))) { + multi = (pmix_multi_event_t*)nxt; + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + PMIX_RETAIN(chain); + chain->multi = multi; + multi->evhdlr(multi->index, + chain->status, &chain->source, + chain->info, chain->ninfo, + chain->results, chain->nresults, + progress_local_event_hdlr, (void*)chain); + goto complete; + } + } + } + /* if we get here, then there are no more multi-mode + * events that match */ + chain->multi = NULL; + /* pickup the beginning of the default event list */ + chain->def = (pmix_default_event_t*)pmix_list_get_begin(&pmix_globals.events.default_events); + } + + /* if they didn't want it to go to a default handler, then we are done */ + if (chain->nondefault) { + goto complete; + } + + if (NULL != chain->def) { + if (pmix_list_get_end(&pmix_globals.events.default_events) != (nxt = pmix_list_get_next(&chain->def->super))) { + def = (pmix_default_event_t*)nxt; + PMIX_RETAIN(chain); + chain->def = def; + def->evhdlr(def->index, + chain->status, &chain->source, + chain->info, chain->ninfo, + chain->results, chain->nresults, + progress_local_event_hdlr, (void*)chain); + } + } + + complete: + /* we still have to call their final callback */ + if (NULL != chain->final_cbfunc) { + chain->final_cbfunc(PMIX_SUCCESS, chain->final_cbdata); + } + /* maintain acctng */ + PMIX_RELEASE(chain); + /* let the caller know that we are done with their callback */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, thiscbdata); + } +} + +/* given notification of an event, cycle thru our list of + * registered callbacks and invoke the matching ones. Note + * that we will invoke the callbacks in order from single + * to multi-event to default, keeping a log of any returned + * info and passing it down to the next invoked event handler. + * Thus, each handler is given the opportunity to see what + * prior handlers did, and decide if anything further needs + * to be done. + */ +void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) +{ + /* We need to parse thru each registered handler and determine + * which one(s) to call for the specific error */ + size_t i; + pmix_single_event_t *sing; + pmix_multi_event_t *multi; + pmix_default_event_t *def; + + /* check for directives */ + if (NULL != chain->info) { + for (i=0; i < chain->ninfo; i++) { + if (0 == strncmp(chain->info[i].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + chain->nondefault = true; + } + } + } + + /* cycle thru the single-event registrations first */ + PMIX_LIST_FOREACH(sing, &pmix_globals.events.single_events, pmix_single_event_t) { + if (sing->code == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + PMIX_RETAIN(chain); + chain->sing = sing; + pmix_output_verbose(2, pmix_globals.debug_output, + "[%s:%d] CALLING SINGLE EVHDLR", + pmix_globals.myid.nspace, pmix_globals.myid.rank); + sing->evhdlr(sing->index, + chain->status, &chain->source, + chain->info, chain->ninfo, + NULL, 0, + progress_local_event_hdlr, (void*)chain); + return; + } + } + + /* if we didn't find any match in the single-event registrations, + * then cycle thru the multi-event registrations next */ + PMIX_LIST_FOREACH(multi, &pmix_globals.events.multi_events, pmix_multi_event_t) { + for (i=0; i < multi->ncodes; i++) { + if (multi->codes[i] == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + PMIX_RETAIN(chain); + chain->multi = multi; + pmix_output_verbose(2, pmix_globals.debug_output, + "[%s:%d] CALLING MULTI EVHDLR", + pmix_globals.myid.nspace, pmix_globals.myid.rank); + multi->evhdlr(multi->index, + chain->status, &chain->source, + chain->info, chain->ninfo, + NULL, 0, + progress_local_event_hdlr, (void*)chain); + return; + } + } + } + + /* if they didn't want it to go to a default handler, then we are done */ + if (chain->nondefault) { + goto complete; + } + + /* finally, pass it to any default handlers */ + PMIX_LIST_FOREACH(def, &pmix_globals.events.default_events, pmix_default_event_t) { + PMIX_RETAIN(chain); + chain->def = def; + pmix_output_verbose(2, pmix_globals.debug_output, + "[%s:%d] CALLING DEFAULT EVHDLR", + pmix_globals.myid.nspace, pmix_globals.myid.rank); + def->evhdlr(def->index, + chain->status, &chain->source, + chain->info, chain->ninfo, + NULL, 0, + progress_local_event_hdlr, (void*)chain); + return; + } + + complete: + /* we still have to call their final callback */ + if (NULL != chain->final_cbfunc) { + chain->final_cbfunc(PMIX_SUCCESS, chain->final_cbdata); + } + return; +} + + +static void _notify_client_event(int sd, short args, void *cbdata) +{ + pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; + pmix_notify_caddy_t *rbout; + pmix_regevents_info_t *reginfoptr; + pmix_peer_events_info_t *pr; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_server: _notify_error notifying clients of error %d", + cd->status); + + /* we cannot know if everyone who wants this notice has had a chance + * to register for it - the notice may be coming too early. So cache + * the message until all local procs have received it, or it ages to + * the point where it gets pushed out by more recent events */ + PMIX_RETAIN(cd); + rbout = pmix_ring_buffer_push(&pmix_server_globals.notifications, cd); + + /* if an older event was bumped, release it */ + if (NULL != rbout) { + PMIX_RELEASE(rbout); + } + + /* cycle across our registered events and send the message to + * any client who registered for it */ + PMIX_LIST_FOREACH(reginfoptr, &pmix_server_globals.events, pmix_regevents_info_t) { + if ((PMIX_MAX_ERR_CONSTANT == reginfoptr->code && !cd->nondefault) || + cd->status == reginfoptr->code) { + PMIX_LIST_FOREACH(pr, ®infoptr->peers, pmix_peer_events_info_t) { + /* if this client was the source of the event, then + * don't send it back */ + if (0 == strncmp(cd->source.nspace, pr->peer->info->nptr->nspace, PMIX_MAX_NSLEN) && + cd->source.rank == pr->peer->info->rank) { + continue; + } + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_server: notifying client %s:%d", + pr->peer->info->nptr->nspace, pr->peer->info->rank); + PMIX_RETAIN(cd->buf); + PMIX_SERVER_QUEUE_REPLY(pr->peer, 0, cd->buf); + } + } + } + + /* notify the caller */ + if (NULL != cd->cbfunc) { + cd->cbfunc(PMIX_SUCCESS, cd->cbdata); + } + PMIX_RELEASE(cd); +} + + +/* as a server, we must do two things: + * + * (a) notify all clients that have registered for this event + * + * (b) callback any of our own functions that have registered + * for this event + */ +static pmix_status_t notify_client_of_event(pmix_status_t status, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_notify_caddy_t *cd; + pmix_cmd_t cmd = PMIX_NOTIFY_CMD; + pmix_status_t rc; + size_t n; + + cd = PMIX_NEW(pmix_notify_caddy_t); + cd->status = status; + if (NULL == source) { + (void)strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); + cd->source.rank = PMIX_RANK_UNDEF; + } else { + (void)strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); + cd->source.rank = source->rank; + } + cd->range = range; + + /* check for directives */ + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + cd->nondefault = true; + } + } + } + + /* pack the command */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &cmd, 1, PMIX_CMD))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + return rc; + } + + /* pack the status */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &status, 1, PMIX_INT))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + return rc; + } + + /* pack the source */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, source, 1, PMIX_PROC))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + return rc; + } + + /* pack any info */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &ninfo, 1, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + return rc; + } + + if (0 < ninfo) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, info, ninfo, PMIX_INFO))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + return rc; + } + } + + /* track the eventual callback info */ + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_server_notify_event status =%d, source = %s:%d, ninfo =%lu", + status, source->nspace, source->rank, ninfo); + + /* we have to push this into our event library to avoid + * potential threading issues */ + PMIX_THREADSHIFT(cd, _notify_client_event); + return PMIX_SUCCESS; +} + +static void sevcon(pmix_single_event_t *p) +{ + p->name = NULL; + p->evhdlr = NULL; +} +static void sevdes(pmix_single_event_t *p) +{ + if (NULL != p->name) { + free(p->name); + } +} +PMIX_CLASS_INSTANCE(pmix_single_event_t, + pmix_list_item_t, + sevcon, sevdes); + +static void mevcon(pmix_multi_event_t *p) +{ + p->name = NULL; + p->codes = NULL; + p->ncodes = 0; + p->evhdlr = NULL; +} +static void mevdes(pmix_multi_event_t *p) +{ + if (NULL != p->name) { + free(p->name); + } + if (NULL != p->codes) { + free(p->codes); + } +} +PMIX_CLASS_INSTANCE(pmix_multi_event_t, + pmix_list_item_t, + mevcon, mevdes); + +static void devcon(pmix_default_event_t *p) +{ + p->name = NULL; + p->evhdlr = NULL; +} +static void devdes(pmix_default_event_t *p) +{ + if (NULL != p->name) { + free(p->name); + } +} +PMIX_CLASS_INSTANCE(pmix_default_event_t, + pmix_list_item_t, + devcon, devdes); + +PMIX_CLASS_INSTANCE(pmix_active_code_t, + pmix_list_item_t, + NULL, NULL); + +static void evcon(pmix_events_t *p) +{ + p->nhdlrs = 0; + PMIX_CONSTRUCT(&p->actives, pmix_list_t); + PMIX_CONSTRUCT(&p->single_events, pmix_list_t); + PMIX_CONSTRUCT(&p->multi_events, pmix_list_t); + PMIX_CONSTRUCT(&p->default_events, pmix_list_t); +} +static void evdes(pmix_events_t *p) +{ + PMIX_LIST_DESTRUCT(&p->actives); + PMIX_LIST_DESTRUCT(&p->single_events); + PMIX_LIST_DESTRUCT(&p->multi_events); + PMIX_LIST_DESTRUCT(&p->default_events); +} +PMIX_CLASS_INSTANCE(pmix_events_t, + pmix_object_t, + evcon, evdes); + +static void chcon(pmix_event_chain_t *p) +{ + memset(p->source.nspace, 0, PMIX_MAX_NSLEN+1); + p->source.rank = PMIX_RANK_UNDEF; + p->nondefault = false; + p->range = PMIX_RANGE_UNDEF; + p->info = NULL; + p->ninfo = 0; + p->results = NULL; + p->nresults = 0; + p->sing = NULL; + p->multi = NULL; + p->def = NULL; + p->final_cbfunc = NULL; + p->final_cbdata = NULL; +} +static void chdes(pmix_event_chain_t *p) +{ + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } + if (NULL != p->results) { + PMIX_INFO_FREE(p->results, p->nresults); + } +} +PMIX_CLASS_INSTANCE(pmix_event_chain_t, + pmix_object_t, + chcon, chdes); diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c new file mode 100644 index 0000000000..51eeb62c9e --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c @@ -0,0 +1,508 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include +#include + +#include +#include +#include + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "src/client/pmix_client_ops.h" +#include "src/server/pmix_server_ops.h" +#include "src/include/pmix_globals.h" + + typedef struct { + pmix_object_t super; + size_t index; + pmix_list_t *list; + pmix_list_item_t *item; + pmix_shift_caddy_t *cd; + pmix_status_t *codes; + size_t ncodes; + pmix_info_t *info; + size_t ninfo; +} pmix_rshift_caddy_t; +static void rscon(pmix_rshift_caddy_t *p) +{ + p->list = NULL; + p->item = NULL; + p->cd = NULL; + p->codes = NULL; + p->ncodes = 0; + p->info = NULL; + p->ninfo = 0; +} +static void rsdes(pmix_rshift_caddy_t *p) +{ + if (NULL != p->cd) { + PMIX_RELEASE(p->cd); + } + if (NULL != p->codes) { + free(p->codes); + } + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } +} +PMIX_CLASS_INSTANCE(pmix_rshift_caddy_t, + pmix_object_t, + rscon, rsdes); + + +static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata) +{ + pmix_rshift_caddy_t *rb = (pmix_rshift_caddy_t*)cbdata; + pmix_status_t rc, ret; + int cnt; + size_t index = rb->index; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: regevents callback recvd"); + + /* unpack the status code */ + cnt = 1; + if ((PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ret, &cnt, PMIX_STATUS))) || + (PMIX_SUCCESS != ret)) { + PMIX_ERROR_LOG(rc); + /* remove the err handler and call the error handler reg completion callback fn.*/ + if (NULL != rb->list && NULL != rb->item) { + pmix_list_remove_item(rb->list, rb->item); + PMIX_RELEASE(rb->item); + } + ret = PMIX_ERR_SERVER_FAILED_REQUEST; + index = UINT_MAX; + } + + /* call the callback */ + if (NULL != rb->cd && NULL != rb->cd->cbfunc.evregcbfn) { + rb->cd->cbfunc.evregcbfn(ret, index, rb->cd->cbdata); + } + PMIX_RELEASE(rb); +} + +static void reg_cbfunc(pmix_status_t status, void *cbdata) +{ + pmix_rshift_caddy_t *rb = (pmix_rshift_caddy_t*)cbdata; + pmix_status_t rc = status; + size_t index = rb->index; + + if (PMIX_SUCCESS != status) { + /* if we failed to register, then remove this event */ + if (NULL != rb->list && NULL != rb->item) { + pmix_list_remove_item(rb->list, rb->item); + PMIX_RELEASE(rb->item); + rc = PMIX_ERR_SERVER_FAILED_REQUEST; + index = UINT_MAX; + } + } + + if (NULL != rb->cd && NULL != rb->cd->cbfunc.evregcbfn) { + /* pass back our local index */ + rb->cd->cbfunc.evregcbfn(rc, index, rb->cd->cbdata); + } + + PMIX_RELEASE(rb); +} + +static pmix_status_t _send_to_server(pmix_rshift_caddy_t *rcd) +{ + pmix_status_t rc; + pmix_buffer_t *msg; + pmix_cmd_t cmd=PMIX_REGEVENTS_CMD; + + msg = PMIX_NEW(pmix_buffer_t); + /* pack the cmd */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* pack the number of codes */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &rcd->cd->ncodes, 1, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* pack any provided codes - may be NULL */ + if (NULL != rcd->cd->codes && 0 < rcd->cd->ncodes) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, rcd->cd->codes, rcd->cd->ncodes, PMIX_STATUS))) { + PMIX_ERROR_LOG(rc); + return rc; + } + } + + /* pack the number of info */ + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &rcd->ninfo, 1, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* pack any provided info - may be NULL */ + if (NULL != rcd->info && 0 < rcd->ninfo) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, rcd->info, rcd->ninfo, PMIX_INFO))) { + PMIX_ERROR_LOG(rc); + return rc; + } + } + PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, regevents_cbfunc, rcd); + + return PMIX_SUCCESS; +} + +static pmix_status_t _add_hdlr(pmix_list_t *list, pmix_list_item_t *item, + size_t index, bool prepend, pmix_list_t *xfer, + pmix_shift_caddy_t *cd) +{ + pmix_rshift_caddy_t *cd2; + pmix_info_caddy_t *ixfer; + size_t n; + bool registered, need_register = false; + pmix_active_code_t *active; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: _add_hdlr"); + + if (prepend) { + pmix_list_prepend(list, item); + } else { + pmix_list_append(list, item); + } + + /* check to see if we have an active registration on these codes */ + if (NULL == cd->codes) { + registered = false; + PMIX_LIST_FOREACH(active, &pmix_globals.events.actives, pmix_active_code_t) { + if (INT_MIN == active->code) { + /* we have registered a default */ + registered = true; + break; + } + } + if (!registered) { + active = PMIX_NEW(pmix_active_code_t); + active->code = INT_MIN; + pmix_list_append(&pmix_globals.events.actives, &active->super); + /* ensure we register it */ + need_register = true; + } + } else { + for (n=0; n < cd->ncodes; n++) { + registered = false; + PMIX_LIST_FOREACH(active, &pmix_globals.events.actives, pmix_active_code_t) { + if (active->code == cd->codes[n]) { + registered = true; + break; + } + } + if (!registered) { + active = PMIX_NEW(pmix_active_code_t); + active->code = cd->codes[n]; + pmix_list_append(&pmix_globals.events.actives, &active->super); + /* ensure we register it */ + need_register = true; + } + } + } + + /* prep next step */ + cd2 = PMIX_NEW(pmix_rshift_caddy_t); + cd2->index = index; + cd2->list = list; + cd2->item = item; + PMIX_RETAIN(cd); + cd2->cd = cd; + cd2->ninfo = pmix_list_get_size(xfer); + PMIX_INFO_CREATE(cd2->info, cd2->ninfo); + n=0; + PMIX_LIST_FOREACH(ixfer, xfer, pmix_info_caddy_t) { + (void)strncpy(cd2->info[n].key, ixfer->info[n].key, PMIX_MAX_KEYLEN); + pmix_value_load(&cd2->info[n].value, &ixfer->info[n].value.data, ixfer->info[n].value.type); + ++n; + } + + /* if we are a client, and we haven't already registered a handler of this + * type with our server, or if we have directives, then we need to notify + * the server */ + if (!pmix_globals.server && + (need_register || 0 < pmix_list_get_size(xfer))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: _add_hdlr sending to server"); + /* send the directives to the server - we will ack this + * registration upon return from there */ + if (PMIX_SUCCESS != (rc = _send_to_server(cd2))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: add_hdlr - pack send_to_server failed status=%d", rc); + PMIX_RELEASE(cd2); + pmix_list_remove_item(list, item); + PMIX_RELEASE(item); + return rc; + } + return PMIX_ERR_WOULD_BLOCK; + } + + /* if we are a server and are registering for events, then we only contact + * our host if we want environmental events */ + + if (pmix_globals.server && cd->enviro && + NULL != pmix_host_server.register_events) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: _add_hdlr registering with server"); + if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(cd->codes, cd->ncodes, + cd2->info, cd2->ninfo, + reg_cbfunc, cd2))) { + PMIX_RELEASE(cd2); + pmix_list_remove_item(list, item); + PMIX_RELEASE(item); + return rc; + } + return PMIX_ERR_WOULD_BLOCK; + } + + return PMIX_SUCCESS; +} + +static void reg_event_hdlr(int sd, short args, void *cbdata) +{ + size_t index = 0, n; + pmix_status_t rc; + pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; + pmix_single_event_t *sing; + pmix_multi_event_t *multi; + pmix_default_event_t *def; + bool prepend = false; + char *name = NULL; + pmix_list_t xfer; + pmix_info_caddy_t *ixfer; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: register event_hdlr with %d infos", (int)cd->ninfo); + + PMIX_CONSTRUCT(&xfer, pmix_list_t); + + /* if directives were included */ + if (NULL != cd->info) { + for (n=0; n < cd->ninfo; n++) { + if (0 == strcmp(cd->info[n].key, PMIX_EVENT_ORDER_PREPEND)) { + /* flag if they asked to prepend this event + * on the precedence order */ + prepend = true; + } else if (0 == strcmp(cd->info[n].key, PMIX_EVENT_HDLR_NAME)) { + name = cd->info[n].value.data.string; + } else if (0 == strcmp(cd->info[n].key, PMIX_EVENT_ENVIRO_LEVEL)) { + cd->enviro = cd->info[n].value.data.flag; + } else { + ixfer = PMIX_NEW(pmix_info_caddy_t); + ixfer->info = &cd->info[n]; + pmix_list_append(&xfer, &ixfer->super); + } + } + } + + /* if the code array is NULL, then this is a default event + * registration request */ + if (NULL == cd->codes) { + def = PMIX_NEW(pmix_default_event_t); + if (NULL != name) { + def->name = strdup(name); + } + index = pmix_globals.events.nhdlrs; + ++pmix_globals.events.nhdlrs; + def->index = index; + def->evhdlr = cd->evhdlr; + rc = _add_hdlr(&pmix_globals.events.default_events, &def->super, + index, prepend, &xfer, cd); + PMIX_LIST_DESTRUCT(&xfer); + if (PMIX_SUCCESS != rc && + PMIX_ERR_WOULD_BLOCK != rc) { + /* unable to register */ + --pmix_globals.events.nhdlrs; + rc = PMIX_ERR_EVENT_REGISTRATION; + index = UINT_MAX; + goto ack; + } + if (PMIX_ERR_WOULD_BLOCK == rc) { + /* the callback will provide our response */ + return; + } + goto ack; + } + + /* if there is only one code, then this is a single event registration */ + if (1 == cd->ncodes) { + sing = PMIX_NEW(pmix_single_event_t); + if (NULL != name) { + sing->name = strdup(name); + } + sing->code = cd->codes[0]; + index = pmix_globals.events.nhdlrs; + sing->index = index; + ++pmix_globals.events.nhdlrs; + rc = _add_hdlr(&pmix_globals.events.single_events, &sing->super, + index, prepend, &xfer, cd); + PMIX_LIST_DESTRUCT(&xfer); + if (PMIX_SUCCESS != rc && + PMIX_ERR_WOULD_BLOCK != rc) { + /* unable to register */ + --pmix_globals.events.nhdlrs; + rc = PMIX_ERR_EVENT_REGISTRATION; + index = UINT_MAX; + goto ack; + } + if (PMIX_ERR_WOULD_BLOCK == rc) { + /* the callback will provide our response */ + return; + } + goto ack; + } + + /* must be a multi-code registration */ + multi = PMIX_NEW(pmix_multi_event_t); + if (NULL != name) { + multi->name = strdup(name); + } + multi->codes = (pmix_status_t*)malloc(cd->ncodes * sizeof(pmix_status_t)); + multi->ncodes = cd->ncodes; + memcpy(multi->codes, cd->codes, cd->ncodes * sizeof(pmix_status_t)); + index = pmix_globals.events.nhdlrs; + multi->index = index; + ++pmix_globals.events.nhdlrs; + rc = _add_hdlr(&pmix_globals.events.multi_events, &multi->super, + index, prepend, &xfer, cd); + PMIX_LIST_DESTRUCT(&xfer); + if (PMIX_SUCCESS != rc && + PMIX_ERR_WOULD_BLOCK != rc) { + /* unable to register */ + --pmix_globals.events.nhdlrs; + rc = PMIX_ERR_EVENT_REGISTRATION; + index = UINT_MAX; + goto ack; + } + if (PMIX_ERR_WOULD_BLOCK == rc) { + /* the callback will provide our response */ + return; + } + + ack: + /* acknowledge the registration so the caller can release + * their data */ + cd->cbfunc.evregcbfn(rc, index, cd->cbdata); + + PMIX_RELEASE(cd); +} + +PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes, + pmix_info_t info[], size_t ninfo, + pmix_notification_fn_t event_hdlr, + pmix_evhdlr_reg_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_shift_caddy_t *cd; + + /* need to thread shift this request so we can access + * our global data to register this *local* event handler */ + cd = PMIX_NEW(pmix_shift_caddy_t); + cd->codes = codes; + cd->ncodes = ncodes; + cd->info = info; + cd->ninfo = ninfo; + cd->evhdlr = event_hdlr; + cd->cbfunc.errregcbfn = cbfunc; + cd->cbdata = cbdata; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_register_event_hdlr shifting to progress thread"); + + PMIX_THREADSHIFT(cd, reg_event_hdlr); +} + +static void dereg_event_hdlr(int sd, short args, void *cbdata) +{ + pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; + pmix_buffer_t *msg; + pmix_single_event_t *sing; + pmix_multi_event_t *multi; + pmix_default_event_t *def; + pmix_cmd_t cmd = PMIX_DEREGEVENTS_CMD; + pmix_status_t rc = PMIX_SUCCESS; + + /* the registration can be in any of three places, so check them all */ + PMIX_LIST_FOREACH(def, &pmix_globals.events.default_events, pmix_default_event_t) { + if (def->index == cd->ref) { + /* found it */ + pmix_list_remove_item(&pmix_globals.events.default_events, &def->super); + PMIX_RELEASE(def); + goto report; + } + } + PMIX_LIST_FOREACH(sing, &pmix_globals.events.single_events, pmix_single_event_t) { + if (sing->index == cd->ref) { + /* found it */ + pmix_list_remove_item(&pmix_globals.events.single_events, &sing->super); + PMIX_RELEASE(sing); + goto report; + } + } + PMIX_LIST_FOREACH(multi, &pmix_globals.events.multi_events, pmix_multi_event_t) { + if (multi->index == cd->ref) { + /* found it */ + pmix_list_remove_item(&pmix_globals.events.multi_events, &multi->super); + PMIX_RELEASE(multi); + goto report; + } + } + /* if we get here, then the registration could not be found */ + if (NULL != cd->cbfunc.opcbfn) { + cd->cbfunc.opcbfn(PMIX_ERR_NOT_FOUND, cd->cbdata); + } + PMIX_RELEASE(cd); + return; + + report: + if (!pmix_globals.server) { + /* notify the server */ + msg = PMIX_NEW(pmix_buffer_t); + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) { + PMIX_RELEASE(msg); + goto cleanup; + } + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cd->ref, 1, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + /* push the message into our event base to send to the server */ + PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, NULL, NULL); + } + + cleanup: + /* must release the caller */ + if (NULL != cd->cbfunc.opcbfn) { + cd->cbfunc.opcbfn(rc, cd->cbdata); + } + PMIX_RELEASE(cd); +} + +PMIX_EXPORT void PMIx_Deregister_event_handler(size_t event_hdlr_ref, + pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_shift_caddy_t *cd; + + /* need to thread shift this request */ + cd = PMIX_NEW(pmix_shift_caddy_t); + cd->cbfunc.opcbfn = cbfunc; + cd->cbdata = cbdata; + cd->ref = event_hdlr_ref; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix_deregister_event_hdlr shifting to progress thread"); + PMIX_THREADSHIFT(cd, dereg_event_hdlr); +} diff --git a/opal/mca/pmix/pmix114/pmix/src/include/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/include/Makefile.am similarity index 96% rename from opal/mca/pmix/pmix114/pmix/src/include/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/include/Makefile.am index 3755203724..a0969fa924 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/include/Makefile.am @@ -37,4 +37,5 @@ headers += \ src/include/prefetch.h \ src/include/types.h +#noinst_include_private_autogen_HEADERS = endif ! PMIX_EMBEDDED_MODE diff --git a/opal/mca/pmix/pmix114/pmix/src/include/align.h b/opal/mca/pmix/pmix2x/pmix/src/include/align.h similarity index 98% rename from opal/mca/pmix/pmix114/pmix/src/include/align.h rename to opal/mca/pmix/pmix2x/pmix/src/include/align.h index 02b361110c..e55c303603 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/align.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/align.h @@ -5,7 +5,7 @@ * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. @@ -13,9 +13,9 @@ * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/opal/mca/pmix/pmix114/pmix/src/include/hash_string.h b/opal/mca/pmix/pmix2x/pmix/src/include/hash_string.h similarity index 99% rename from opal/mca/pmix/pmix114/pmix/src/include/hash_string.h rename to opal/mca/pmix/pmix2x/pmix/src/include/hash_string.h index 5d1e784979..a079f793ca 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/hash_string.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/hash_string.h @@ -3,9 +3,9 @@ * of Tennessee Research Foundation. All rights * reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/opal/mca/pmix/pmix114/pmix/src/include/pmix_config.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_config.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/include/pmix_config.h rename to opal/mca/pmix/pmix2x/pmix/src/include/pmix_config.h diff --git a/opal/mca/pmix/pmix114/pmix/src/include/pmix_globals.c b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c similarity index 89% rename from opal/mca/pmix/pmix114/pmix/src/include/pmix_globals.c rename to opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c index db040c9aef..45b9fc7dde 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/pmix_globals.c +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . @@ -57,7 +57,7 @@ void pmix_globals_init(void) { memset(&pmix_globals.myid, 0, sizeof(pmix_proc_t)); PMIX_CONSTRUCT(&pmix_globals.nspaces, pmix_list_t); - pmix_pointer_array_init(&pmix_globals.errregs, 1, PMIX_MAX_ERROR_REGISTRATIONS, 1); + PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t); } void pmix_globals_finalize(void) @@ -69,6 +69,7 @@ void pmix_globals_finalize(void) if (NULL != pmix_globals.cache_remote) { PMIX_RELEASE(pmix_globals.cache_remote); } + PMIX_DESTRUCT(&pmix_globals.events); } @@ -157,29 +158,25 @@ PMIX_CLASS_INSTANCE(pmix_rank_info_t, pmix_list_item_t, info_con, info_des); -static void errcon(pmix_error_reg_info_t *p) -{ - p->errhandler = NULL; - p->info = NULL; - p->ninfo = 0; -} -static void errdes(pmix_error_reg_info_t *p) -{ - p->errhandler = NULL; - // PMIX_INFO_FREE(p->info, p->ninfo); -} -PMIX_CLASS_INSTANCE(pmix_error_reg_info_t, - pmix_object_t, - errcon, errdes); - static void scon(pmix_shift_caddy_t *p) { p->active = false; + p->codes = NULL; + p->ncodes = 0; + p->nspace = NULL; + p->data = NULL; + p->ndata = 0; + p->info = NULL; + p->ninfo = 0; + p->evhdlr = NULL; p->kv = NULL; + p->vptr = NULL; + p->cd = NULL; + p->tracker = NULL; + p->enviro = false; p->cbfunc.relfn = NULL; - p->cbfunc.errregcbfn = NULL; - p->cbfunc.opcbfn = NULL; p->cbdata = NULL; + p->ref = 0; } static void scdes(pmix_shift_caddy_t *p) { @@ -190,3 +187,7 @@ static void scdes(pmix_shift_caddy_t *p) PMIX_CLASS_INSTANCE(pmix_shift_caddy_t, pmix_object_t, scon, scdes); + +PMIX_CLASS_INSTANCE(pmix_info_caddy_t, + pmix_list_item_t, + NULL, NULL); diff --git a/opal/mca/pmix/pmix114/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h similarity index 92% rename from opal/mca/pmix/pmix114/pmix/src/include/pmix_globals.h rename to opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h index 3f17a8fce6..252fd0bf7a 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h @@ -35,11 +35,14 @@ #include "src/buffer_ops/types.h" #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" +#include "src/event/pmix_event.h" BEGIN_C_DECLS -#define PMIX_MAX_CRED_SIZE 131072 // set max at 128kbytes -#define PMIX_MAX_ERROR_REGISTRATIONS 128 // maximum number of error handlers that can be registered +/* some limits */ +#define PMIX_MAX_CRED_SIZE 131072 // set max at 128kbytes +#define PMIX_MAX_ERR_CONSTANT INT_MIN + /**** ENUM DEFINITIONS ****/ /* define a command type for communicating to the @@ -186,16 +189,6 @@ typedef struct pmix_peer_t { PMIX_CLASS_DECLARATION(pmix_peer_t); -/* define a structure for tracking error registrations */ -typedef struct { - pmix_object_t super; - bool sglhdlr; // registers a specific error status handler - pmix_notification_fn_t errhandler; /* registered err handler callback fn */ - pmix_info_t *info; /* error info keys registered with the handler */ - size_t ninfo; /* size of info */ -} pmix_error_reg_info_t; -PMIX_CLASS_DECLARATION(pmix_error_reg_info_t); - typedef struct { pmix_list_item_t super; char *name; // name of the node @@ -249,6 +242,8 @@ PMIX_CLASS_DECLARATION(pmix_server_trkr_t); pmix_event_t ev; volatile bool active; pmix_status_t status; + pmix_status_t *codes; + size_t ncodes; const char *nspace; int rank; const char *data; @@ -256,21 +251,31 @@ PMIX_CLASS_DECLARATION(pmix_server_trkr_t); const char *key; pmix_info_t *info; size_t ninfo; - pmix_notification_fn_t err; + pmix_notification_fn_t evhdlr; pmix_kval_t *kv; pmix_value_t *vptr; pmix_server_caddy_t *cd; pmix_server_trkr_t *tracker; + bool enviro; union { pmix_release_cbfunc_t relfn; - pmix_errhandler_reg_cbfunc_t errregcbfn; + pmix_evhdlr_reg_cbfunc_t evregcbfn; pmix_op_cbfunc_t opcbfn; + pmix_evhdlr_reg_cbfunc_t errregcbfn; }cbfunc; void *cbdata; - int ref; + size_t ref; } pmix_shift_caddy_t; PMIX_CLASS_DECLARATION(pmix_shift_caddy_t); +/* define a very simple caddy for dealing with pmix_info_t + * objects when transferring portions of arrays */ +typedef struct { + pmix_list_item_t super; + pmix_info_t *info; +} pmix_info_caddy_t; +PMIX_CLASS_DECLARATION(pmix_info_caddy_t); + #define PMIX_THREADSHIFT(r, c) \ do { \ (r)->active = true; \ @@ -301,7 +306,7 @@ typedef struct { pmix_event_base_t *evbase; bool external_evbase; int debug_output; - pmix_pointer_array_t errregs; // my error handler registrations. + pmix_events_t events; // my event handler registrations. bool server; bool connected; pmix_list_t nspaces; // list of pmix_nspace_t for the nspaces we know about diff --git a/opal/mca/pmix/pmix114/pmix/src/include/pmix_socket_errno.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_socket_errno.h similarity index 98% rename from opal/mca/pmix/pmix114/pmix/src/include/pmix_socket_errno.h rename to opal/mca/pmix/pmix2x/pmix/src/include/pmix_socket_errno.h index 38b1c367b0..2b7ecb506c 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/pmix_socket_errno.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_socket_errno.h @@ -5,15 +5,15 @@ * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Intel, Inc. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ #ifndef PMIX_GET_SOCKET_ERROR_H diff --git a/opal/mca/pmix/pmix114/pmix/src/include/prefetch.h b/opal/mca/pmix/pmix2x/pmix/src/include/prefetch.h similarity index 99% rename from opal/mca/pmix/pmix114/pmix/src/include/prefetch.h rename to opal/mca/pmix/pmix2x/pmix/src/include/prefetch.h index 77db04a16a..78fc3f49ae 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/prefetch.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/prefetch.h @@ -3,9 +3,9 @@ * All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/opal/mca/pmix/pmix114/pmix/src/include/types.h b/opal/mca/pmix/pmix2x/pmix/src/include/types.h similarity index 98% rename from opal/mca/pmix/pmix114/pmix/src/include/types.h rename to opal/mca/pmix/pmix2x/pmix/src/include/types.h index 5916064edb..0bc5dc2dd7 100644 --- a/opal/mca/pmix/pmix114/pmix/src/include/types.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/types.h @@ -5,7 +5,7 @@ * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. @@ -13,9 +13,9 @@ * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -160,7 +160,7 @@ static inline void* pmix_ptr_ltop( uint64_t value ) #if defined(WORDS_BIGENDIAN) || !defined(HAVE_UNIX_BYTESWAP) static inline uint16_t pmix_swap_bytes2(uint16_t val) __pmix_attribute_const__; -static inline uint16_t pmix_swap_bytes2(uint16_t val) +static inline uint16_t pmix_swap_bytes2(uint16_t val) { union { uint16_t bigval; uint8_t arrayval[2]; @@ -205,7 +205,7 @@ static inline uint64_t pmix_swap_bytes8(uint64_t val) r.arrayval[5] = w.arrayval[2]; r.arrayval[6] = w.arrayval[1]; r.arrayval[7] = w.arrayval[0]; - + return r.bigval; } diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/sec/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/sec/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_munge.c b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_munge.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_munge.c rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_munge.c diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_munge.h b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_munge.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_munge.h rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_munge.h diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_native.c b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_native.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_native.c rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_native.c index 677a07dd45..a4dd549a02 100644 --- a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_native.c +++ b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_native.c @@ -1,6 +1,7 @@ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. + * * $COPYRIGHT$ * * Additional copyrights may follow @@ -15,6 +16,7 @@ #include "src/include/pmix_socket_errno.h" #include "src/include/pmix_globals.h" #include "src/util/argv.h" +#include "src/util/error.h" #include "src/util/output.h" #include "src/usock/usock.h" diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_native.h b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_native.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_native.h rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_native.h diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_sasl.c b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sasl.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_sasl.c rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sasl.c diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_sasl.h b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sasl.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_sasl.h rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sasl.h diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_sec.c b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sec.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_sec.c rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sec.c diff --git a/opal/mca/pmix/pmix114/pmix/src/sec/pmix_sec.h b/opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sec.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/sec/pmix_sec.h rename to opal/mca/pmix/pmix2x/pmix/src/sec/pmix_sec.h diff --git a/opal/mca/pmix/pmix114/pmix/src/server/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/server/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/server/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/server/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c similarity index 82% rename from opal/mca/pmix/pmix114/pmix/src/server/pmix_server.c rename to opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c index 2626dbeb61..0a062ad04d 100644 --- a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c @@ -55,6 +55,9 @@ #include "src/util/progress_threads.h" #include "src/usock/usock.h" #include "src/sec/pmix_sec.h" +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) +#include "src/dstore/pmix_dstore.h" +#endif /* PMIX_ENABLE_DSTORE */ #include "pmix_server_ops.h" @@ -62,24 +65,12 @@ pmix_server_globals_t pmix_server_globals = {{{0}}}; // local variables -static char *myuri = NULL; -static struct sockaddr_un myaddress; static char *security_mode = NULL; // local functions for connection support static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); - -typedef struct { - pmix_object_t super; - pmix_event_t ev; - pmix_peer_t *peer; - pmix_buffer_t *buf; - uint32_t tag; -} pmix_usock_queue_t; -PMIX_CLASS_INSTANCE(pmix_usock_queue_t, - pmix_object_t, - NULL, NULL); +static inline int _my_client(const char *nspace, int rank); /* queue a message to be sent to one of our procs - must * provide the following params: @@ -88,7 +79,7 @@ PMIX_CLASS_INSTANCE(pmix_usock_queue_t, * t - tag to be sent to * b - buffer to be sent */ -static void _queue_message(int fd, short args, void *cbdata) +void pmix_server_queue_message(int fd, short args, void *cbdata) { pmix_usock_queue_t *queue = (pmix_usock_queue_t*)cbdata; pmix_usock_send_t *snd; @@ -121,31 +112,13 @@ static void _queue_message(int fd, short args, void *cbdata) PMIX_RELEASE(queue); } -#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \ - do { \ - pmix_usock_queue_t *queue; \ - queue = PMIX_NEW(pmix_usock_queue_t); \ - queue->peer = (p); \ - queue->buf = (b); \ - queue->tag = (t); \ - pmix_output_verbose(2, pmix_globals.debug_output, \ - "[%s:%d] queue reply to %s:%d on tag %d", \ - __FILE__, __LINE__, \ - (queue->peer)->info->nptr->nspace, \ - (queue->peer)->info->rank, (queue->tag)); \ - event_assign(&queue->ev, pmix_globals.evbase, -1, \ - EV_WRITE, _queue_message, queue); \ - event_priority_set(&queue->ev, 0); \ - event_active(&queue->ev, EV_WRITE, 1); \ - } while (0) - - static pmix_status_t initialize_server_base(pmix_server_module_t *module) { int debug_level; char *tdir, *evar; pid_t pid; char * pmix_pid; + pmix_listener_t *listener; /* initialize the output system */ if (!pmix_output_init()) { @@ -154,7 +127,6 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module) /* setup the globals */ pmix_globals_init(); memset(&pmix_server_globals, 0, sizeof(pmix_server_globals)); - pmix_server_globals.listen_socket = -1; /* mark that I am a server */ pmix_globals.server = true; @@ -183,9 +155,12 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module) pmix_pointer_array_init(&pmix_server_globals.clients, 1, INT_MAX, 1); PMIX_CONSTRUCT(&pmix_server_globals.collectives, pmix_list_t); PMIX_CONSTRUCT(&pmix_server_globals.remote_pnd, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.client_eventregs, pmix_list_t); PMIX_CONSTRUCT(&pmix_server_globals.gdata, pmix_buffer_t); + PMIX_CONSTRUCT(&pmix_server_globals.events, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.notifications, pmix_ring_buffer_t); + PMIX_CONSTRUCT(&pmix_server_globals.listeners, pmix_list_t); + pmix_ring_buffer_init(&pmix_server_globals.notifications, 256); /* see if debug is requested */ if (NULL != (evar = getenv("PMIX_DEBUG"))) { @@ -216,29 +191,30 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module) } } - /* now set the address - we use the pid here to reduce collisions */ - memset(&myaddress, 0, sizeof(struct sockaddr_un)); - myaddress.sun_family = AF_UNIX; - if (0 > asprintf(&pmix_pid, "pmix-%d", pid)) { + /* for now, just setup the v1.1 series rendezvous point + * we use the pid to reduce collisions */ + if (0 > asprintf(&pmix_pid, "%s/pmix-%d", tdir, pid)) { return PMIX_ERR_NOMEM; } - // If the above set temporary directory name plus the pmix-PID string - // plus the '/' separator are too long, just fail, so the caller - // may provide the user with a proper help... *Cough*, *Cough* OSX... - if ((strlen(tdir) + strlen(pmix_pid) + 1) > sizeof(myaddress.sun_path)-1) { + if ((strlen(pmix_pid) + 1) > sizeof(listener->address.sun_path)-1) { free(pmix_pid); return PMIX_ERR_INVALID_LENGTH; } - snprintf(myaddress.sun_path, sizeof(myaddress.sun_path)-1, "%s/%s", tdir, pmix_pid); - free(pmix_pid); - if (0 > asprintf(&myuri, "%s:%lu:%s", pmix_globals.myid.nspace, - (unsigned long)pmix_globals.myid.rank, myaddress.sun_path)) { + + listener = PMIX_NEW(pmix_listener_t); + snprintf(listener->address.sun_path, sizeof(listener->address.sun_path)-1, "%s", pmix_pid); + if (0 > asprintf(&listener->uri, "%s:%lu:%s", pmix_globals.myid.nspace, + (unsigned long)pmix_globals.myid.rank, listener->address.sun_path)) { + free(pmix_pid); return PMIX_ERR_NOMEM; } - + listener->varname = strdup("PMIX_SERVER_URI"); + listener->protocol_type = 1; + pmix_list_append(&pmix_server_globals.listeners, &listener->super); + free(pmix_pid); pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:server constructed uri %s", myuri); + "pmix:server constructed uri %s", listener->uri); return PMIX_SUCCESS; } @@ -250,9 +226,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_status_t rc; size_t n; pmix_kval_t kv; - uid_t sockuid = -1; - gid_t sockgid = -1; - mode_t sockmode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH; + pmix_listener_t *lt; ++pmix_globals.init_cntr; if (1 < pmix_globals.init_cntr) { @@ -266,6 +240,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, return rc; } +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + if (PMIX_SUCCESS != (rc = pmix_dstore_init())) { + return rc; + } +#endif /* PMIX_ENABLE_DSTORE */ + /* and the usock system */ pmix_usock_init(NULL); @@ -280,12 +260,21 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, for (n=0; n < ninfo; n++) { if (0 == strcmp(info[n].key, PMIX_USERID)) { /* the userid is in the uint32_t storage */ - sockuid = info[n].value.data.uint32; + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + lt->owner = info[n].value.data.uint32; + lt->owner_given = true; + } } else if (0 == strcmp(info[n].key, PMIX_GRPID)) { /* the grpid is in the uint32_t storage */ - sockgid = info[n].value.data.uint32; + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + lt->group = info[n].value.data.uint32; + lt->group_given = true; + } } else if (0 == strcmp(info[n].key, PMIX_SOCKET_MODE)) { - sockmode = info[n].value.data.uint32 & 0777; + /* socket mode is in the uint32_t storage */ + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + lt->mode = info[n].value.data.uint32; + } } } } @@ -298,9 +287,11 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_list_append(&pmix_usock_globals.posted_recvs, &req->super); /* start listening */ - if (PMIX_SUCCESS != pmix_start_listening(&myaddress, sockmode, sockuid, sockgid)) { - PMIx_server_finalize(); - return PMIX_ERR_INIT; + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + if (PMIX_SUCCESS != pmix_start_listening(lt)) { + PMIx_server_finalize(); + return PMIX_ERR_INIT; + } } /* check the info keys for info we @@ -346,12 +337,9 @@ static void cleanup_server_state(void) PMIX_LIST_DESTRUCT(&pmix_server_globals.collectives); PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd); PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs); - PMIX_LIST_DESTRUCT(&pmix_server_globals.client_eventregs); PMIX_DESTRUCT(&pmix_server_globals.gdata); + PMIX_LIST_DESTRUCT(&pmix_server_globals.listeners); - if (NULL != myuri) { - free(myuri); - } if (NULL != security_mode) { free(security_mode); } @@ -388,14 +376,11 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) libevent_global_shutdown(); #endif - if (0 <= pmix_server_globals.listen_socket) { - CLOSE_THE_SOCKET(pmix_server_globals.listen_socket); - } - pmix_usock_finalize(); - /* cleanup the rendezvous file */ - unlink(myaddress.sun_path); +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + pmix_dstore_finalize(); +#endif /* PMIX_ENABLE_DSTORE */ cleanup_server_state(); pmix_output_verbose(2, pmix_globals.debug_output, @@ -594,43 +579,30 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n static void _deregister_nspace(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_nspace_t *nptr; - int i; - pmix_peer_t *peer; + pmix_nspace_t *tmp; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:server _deregister_nspace %s", cd->proc.nspace); /* see if we already have this nspace */ - PMIX_LIST_FOREACH(nptr, &pmix_globals.nspaces, pmix_nspace_t) { - if (0 == strcmp(nptr->nspace, cd->proc.nspace)) { - /* find and remove this client from our array of local - * peers - remember that it can occur multiple times - * if the peer called fork/exec and its children called - * PMIx_Init! We have to rely on none of those children - * living beyond our child as we otherwise cannot - * track them */ - for (i=0; i < pmix_server_globals.clients.size; i++) { - if (NULL == (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { - continue; - } - if (nptr == peer->info->nptr) { - /* remove this entry */ - pmix_pointer_array_set_item(&pmix_server_globals.clients, i, NULL); - PMIX_RELEASE(peer); - } - } - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); + PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { + pmix_list_remove_item(&pmix_globals.nspaces, &tmp->super); + PMIX_RELEASE(tmp); break; } } + if (NULL != cd->opcbfunc) { + cd->opcbfunc(PMIX_SUCCESS, cd->cbdata); + } PMIX_RELEASE(cd); } -PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[]) +PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], + pmix_op_cbfunc_t cbfunc, + void *cbdata) { pmix_setup_caddy_t *cd; @@ -640,6 +612,8 @@ PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[]) cd = PMIX_NEW(pmix_setup_caddy_t); (void)strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; /* we have to push this into our event library to avoid * potential threading issues */ @@ -872,9 +846,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, static void _deregister_client(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; + pmix_rank_info_t *info; pmix_nspace_t *nptr, *tmp; - int i; - pmix_peer_t *peer; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:server _deregister_client for nspace %s rank %d", @@ -892,32 +865,24 @@ static void _deregister_client(int sd, short args, void *cbdata) /* nothing to do */ goto cleanup; } - /* find and remove this client from our array of local - * peers - remember that it can occur multiple times - * if the peer called fork/exec and its children called - * PMIx_Init! We have to rely on none of those children - * living beyond our child as we otherwise cannot - * track them */ - for (i=0; i < pmix_server_globals.clients.size; i++) { - if (NULL == (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { - continue; - } - if (nptr != peer->info->nptr) { - continue; - } - if (cd->proc.rank == peer->info->rank) { - /* remove this entry */ - pmix_pointer_array_set_item(&pmix_server_globals.clients, i, NULL); - PMIX_RELEASE(peer); + /* find an remove this client */ + PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) { + if (info->rank == cd->proc.rank) { + pmix_list_remove_item(&nptr->server->ranks, &info->super); + PMIX_RELEASE(info); + break; } } - cleanup: + if (NULL != cd->opcbfunc) { + cd->opcbfunc(PMIX_SUCCESS, cd->cbdata); + } PMIX_RELEASE(cd); } -PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc) +PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; @@ -928,6 +893,8 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc) cd = PMIX_NEW(pmix_setup_caddy_t); (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; /* we have to push this into our event library to avoid * potential threading issues */ @@ -938,6 +905,7 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc) PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char ***env) { char rankstr[128]; + pmix_listener_t *lt; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:server setup_fork for nspace %s rank %d", @@ -949,7 +917,9 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char * (void)snprintf(rankstr, 127, "%d", proc->rank); pmix_setenv("PMIX_RANK", rankstr, true, env); /* pass our rendezvous info */ - pmix_setenv("PMIX_SERVER_URI", myuri, true, env); + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + pmix_setenv(lt->varname, lt->uri, true, env); + } /* pass our active security mode */ pmix_setenv("PMIX_SECURITY_MODE", security_mode, true, env); @@ -1083,283 +1053,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_dmodex_request(const pmix_proc_t *proc, return PMIX_SUCCESS; } -static bool match_error_registration(pmix_regevents_info_t *reginfoptr, pmix_notify_caddy_t *cd) -{ - unsigned int i, j; - char errgroup[PMIX_MAX_KEYLEN]; - pmix_info_t *info = reginfoptr->info; - size_t ninfo = reginfoptr->ninfo; - pmix_status_t error = cd->status; - - if (NULL == info || ninfo <= 0) { - /* this is a general errhandler, and so it always matches. - * however, here we are looking for an exact match, and - * so we ignore general errhandlers unless the incoming - * one is also general */ - if (NULL == cd->info || 0 == cd->ninfo) { - return true; - } else { - return false; - } - } - - /* since this errhandler has info keys, it is not a general errhandler. - * If the incoming errhandler *is* a general one, then we must not - * match so we can store the general case */ - if (NULL == cd->info || 0 == cd->ninfo) { - return false; - } - - /* try to match using error name or error group keys - this indicates - * a request for a specific error state */ - pmix_get_errorgroup(error, errgroup); - for (i=0; i < ninfo; i++) { - // if we get a match on any key then we abort the search and return true. - if ((0 == strncmp(info[i].key, PMIX_ERROR_NAME, PMIX_MAX_KEYLEN)) && - (error == info[i].value.data.int32)) { - return true; - } else if ((0 == strncmp(info[i].key, errgroup, PMIX_MAX_KEYLEN)) && - (true == info[i].value.data.flag)) { - return true; - } - } - - /* if we get here, then they haven't asked for a specific error state. - * It is possible, however, that they are asking for all errors from a - * specific node, so search by node (error location) key if it is - * specified in the notify info list */ - for (i=0; i < cd->ninfo ; i++) { - if (0 == strncmp(cd->info[i].key, PMIX_ERROR_NODE_NAME, PMIX_MAX_KEYLEN)) { - for (j=0; j < ninfo; j++) { - if ((0 == strncmp(info[j].key, PMIX_ERROR_NODE_NAME, PMIX_MAX_KEYLEN)) && - (0 == strcmp(info[j].value.data.string, cd->info[i].value.data.string))) { - return true; - } - } - } - } - - /* end of search and nothing matched, so return false */ - return false; -} - -static void _notify_error(int sd, short args, void *cbdata) -{ - pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; - pmix_status_t rc; - pmix_cmd_t cmd = PMIX_NOTIFY_CMD; - int i; - size_t j; - pmix_peer_t *peer; - pmix_regevents_info_t *reginfoptr; - bool notify, notifyall; - - pmix_output_verbose(0, pmix_globals.debug_output, - "pmix_server: _notify_error notifying client of error %d", - cd->status); - - /* pack the command */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &cmd, 1, PMIX_CMD))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - /* pack the status */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &cd->status, 1, PMIX_INT))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - /* pack the error procs */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &cd->error_nprocs, 1, PMIX_SIZE))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - if (0 < cd->error_nprocs) { - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, cd->error_procs, cd->error_nprocs, PMIX_PROC))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - } - - /* pack the info */ - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, &cd->ninfo, 1, PMIX_SIZE))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - if (0 < cd->ninfo) { - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(cd->buf, cd->info, cd->ninfo, PMIX_INFO))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - } - - /* if the RM gave us a NULL proc list, then we are notifying everyone */ - if (NULL == cd->procs) { - notifyall = true; - } else { - notifyall = false; - } - - /* cycle across our connected clients and send the message to - * any within the specified proc array */ - for (i=0; i < pmix_server_globals.clients.size; i++) { - if (NULL == (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { - continue; - } - if (!notifyall) { - /* check to see if this proc matches that of one in the specified array */ - notify = false; - for (j=0; j < cd->nprocs; j++) { - if (0 != strncmp(peer->info->nptr->nspace, cd->procs[j].nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == cd->procs[j].rank || - cd->procs[j].rank == peer->info->rank) { - notify = true; - break; - } - } - if (!notify) { - /* if we are not notifying everyone, and this proc isn't to - * be notified, then just continue the main loop */ - continue; - } - } - - /* get the client's error registration and check if client - * requested notification of this error */ - reginfoptr = NULL; - notify = false; - PMIX_LIST_FOREACH(reginfoptr, &pmix_server_globals.client_eventregs, pmix_regevents_info_t) { - if (reginfoptr->peer == peer) { - /* check if the client has registered for this error - * by parsing the info keys */ - notify = match_error_registration(reginfoptr, cd); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server _notify_error - match error registration returned notify =%d ", notify); - } - if (notify) { - break; - } - } - if (notify) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server: _notify_error - notifying process rank %d error %d", - peer->info->rank, cd->status); - PMIX_RETAIN(cd->buf); - PMIX_SERVER_QUEUE_REPLY(peer, 0, cd->buf); - } - } - - cleanup: - /* notify the caller */ - if (NULL != cd->cbfunc) { - cd->cbfunc(rc, cd->cbdata); - } - PMIX_RELEASE(cd); -} - -pmix_status_t pmix_server_notify_error(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_proc_t error_procs[], size_t error_nprocs, - pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) -{ - pmix_notify_caddy_t *cd; - - cd = PMIX_NEW(pmix_notify_caddy_t); - cd->status = status; - cd->procs = procs; - cd->nprocs = nprocs; - cd->error_procs = error_procs; - cd->error_nprocs = error_nprocs; - cd->info = info; - cd->ninfo = ninfo; - cd->cbfunc = cbfunc; - cd->cbdata = cbdata; - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server_notify_error status =%d, nprocs = %lu, ninfo =%lu", - status, nprocs, ninfo); - - /* we have to push this into our event library to avoid - * potential threading issues */ - PMIX_THREADSHIFT(cd, _notify_error); - return PMIX_SUCCESS; -} - -static void reg_errhandler(int sd, short args, void *cbdata) -{ - int index = 0; - pmix_status_t rc; - pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; - - /* check if this handler is already registered if so return error */ - if (PMIX_SUCCESS == pmix_lookup_errhandler(cd->err, &index)) { - /* complete request with error status and return its original reference */ - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server_register_errhandler error - hdlr already registered index = %d", - index); - cd->cbfunc.errregcbfn(PMIX_EXISTS, index, cd->cbdata); - } else { - rc = pmix_add_errhandler(cd->err, cd->info, cd->ninfo, &index); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server_register_errhandler - success index =%d", index); - cd->cbfunc.errregcbfn(rc, index, cd->cbdata); - } - cd->active = false; - PMIX_RELEASE(cd); -} - -void pmix_server_register_errhandler(pmix_info_t info[], size_t ninfo, - pmix_notification_fn_t errhandler, - pmix_errhandler_reg_cbfunc_t cbfunc, - void *cbdata) -{ - pmix_shift_caddy_t *cd; - - /* need to thread shift this request */ - cd = PMIX_NEW(pmix_shift_caddy_t); - cd->info = info; - cd->ninfo = ninfo; - cd->err = errhandler; - cd->cbfunc.errregcbfn = cbfunc; - cd->cbdata = cbdata; - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server_register_errhandler shifting to server thread"); - - PMIX_THREADSHIFT(cd, reg_errhandler); -} - -static void dereg_errhandler(int sd, short args, void *cbdata) -{ - pmix_status_t rc; - pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; - - rc = pmix_remove_errhandler(cd->ref); - if (NULL != cd->cbfunc.opcbfn) { - cd->cbfunc.opcbfn(rc, cd->cbdata); - } - cd->active = false; -} - -void pmix_server_deregister_errhandler(int errhandler_ref, - pmix_op_cbfunc_t cbfunc, - void *cbdata) -{ - pmix_shift_caddy_t *cd; - - /* need to thread shift this request */ - cd = PMIX_NEW(pmix_shift_caddy_t); - cd->cbfunc.opcbfn = cbfunc; - cd->cbdata = cbdata; - cd->ref = errhandler_ref; - PMIX_THREADSHIFT(cd, dereg_errhandler); - - PMIX_WAIT_FOR_COMPLETION(cd->active); - PMIX_RELEASE(cd); - } - static void _store_internal(int sd, short args, void *cbdata) { pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; @@ -1983,15 +1676,24 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) /* there may be multiple blobs for this rank, each from a different scope */ cnt = 1; while (PMIX_SUCCESS == (rc = pmix_bfrop.unpack(bptr, &bpscope, &cnt, PMIX_BUFFER))) { + /* don't store blobs to the sm dstore from local clients */ + if (_my_client(nptr->nspace, rank)) { + continue; + } pmix_kval_t *kp = PMIX_NEW(pmix_kval_t); kp->key = strdup("modex"); PMIX_VALUE_CREATE(kp->value, 1); kp->value->type = PMIX_BYTE_OBJECT; PMIX_UNLOAD_BUFFER(bpscope, kp->value->data.bo.bytes, kp->value->data.bo.size); /* store it in the appropriate hash */ - if (PMIX_SUCCESS != (rc = pmix_hash_store(&nptr->server->remote, rank, kp))) { + if (PMIX_SUCCESS != (rc = pmix_hash_store(&nptr->server->remote, rank, kp))) { PMIX_ERROR_LOG(rc); } +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + if (PMIX_SUCCESS != (rc = pmix_dstore_store(nptr->nspace, rank, kp))) { + PMIX_ERROR_LOG(rc); + } +#endif /* PMIX_ENABLE_DSTORE */ PMIX_RELEASE(kp); // maintain acctg } // while bpscope if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { @@ -2217,29 +1919,17 @@ static void cnct_cbfunc(pmix_status_t status, void *cbdata) PMIX_THREADSHIFT(scd, _cnct); } -void regevents_cbfunc (pmix_status_t status, void *cbdata) +static void regevents_cbfunc (pmix_status_t status, void *cbdata) { pmix_status_t rc; pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata; - pmix_regevents_info_t *reginfo, *reginfo_next; pmix_buffer_t *reply; + pmix_output_verbose(2, pmix_globals.debug_output, "server:regevents_cbfunc called status = %d", status); - if (PMIX_SUCCESS != status) { - /* need to delete the stored event reg info when server - nacks reg events request */ - PMIX_LIST_FOREACH_SAFE(reginfo, reginfo_next, &pmix_server_globals.client_eventregs, - pmix_regevents_info_t) { - if(reginfo->peer == cd->peer) { - pmix_list_remove_item (&pmix_server_globals.client_eventregs, - ®info->super); - PMIX_RELEASE(reginfo); - break; - } - } - } + reply = PMIX_NEW(pmix_buffer_t); - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_INT))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_STATUS))) { PMIX_ERROR_LOG(rc); } // send reply @@ -2252,10 +1942,13 @@ static void deregevents_cbfunc (pmix_status_t status, void *cbdata) pmix_status_t rc; pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata; pmix_buffer_t *reply = PMIX_NEW(pmix_buffer_t); + pmix_output_verbose(2, pmix_globals.debug_output, "server:deregevents_cbfunc called status = %d", status); - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_INT))) + + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_STATUS))) { PMIX_ERROR_LOG(rc); + } // send reply PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); PMIX_RELEASE(cd); @@ -2266,10 +1959,13 @@ static void notifyerror_cbfunc (pmix_status_t status, void *cbdata) pmix_status_t rc; pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata; pmix_buffer_t *reply = PMIX_NEW(pmix_buffer_t); + pmix_output_verbose(2, pmix_globals.debug_output, "server:notifyerror_cbfunc called status = %d", status); - if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_INT))) + + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_STATUS))) { PMIX_ERROR_LOG(rc); + } // send reply PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); PMIX_RELEASE(cd); @@ -2440,7 +2136,7 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, } if (PMIX_NOTIFY_CMD == cmd) { PMIX_PEER_CADDY(cd, peer, tag); - rc = pmix_server_notify_error_client(peer, buf, notifyerror_cbfunc, cd); + rc = pmix_server_event_recvd_from_client(peer, buf, notifyerror_cbfunc, cd); return rc; } return PMIX_ERR_NOT_SUPPORTED; @@ -2466,3 +2162,21 @@ static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr PMIX_SERVER_QUEUE_REPLY(peer, hdr->tag, reply); } } + +static inline int _my_client(const char *nspace, int rank) +{ + pmix_peer_t *peer; + int i; + int local = 0; + + for (i = 0; i < pmix_server_globals.clients.size; i++) { + if (NULL != (peer = (pmix_peer_t *)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { + if (0 == strcmp(peer->info->nptr->nspace, nspace) && peer->info->rank == rank) { + local = 1; + break; + } + } + } + + return local; +} diff --git a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_get.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c similarity index 96% rename from opal/mca/pmix/pmix114/pmix/src/server/pmix_server_get.c rename to opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c index 515d5d0dd4..971b2ce757 100644 --- a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_get.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c @@ -54,6 +54,9 @@ #include "src/util/progress_threads.h" #include "src/usock/usock.h" #include "src/sec/pmix_sec.h" +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) +#include "src/dstore/pmix_dstore.h" +#endif /* PMIX_ENABLE_DSTORE */ #include "pmix_server_ops.h" @@ -361,7 +364,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, int rank, size_t sz; int cur_rank; int found = 0; - pmix_buffer_t xfer, pbkt, *xptr; + pmix_buffer_t pbkt; void *last; pmix_hash_table_t *hts[3]; pmix_hash_table_t **htptr; @@ -410,6 +413,20 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, int rank, } while (PMIX_SUCCESS == rc) { if (NULL != val) { +#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) + pmix_kval_t *kv; + + /* setup to xfer the data */ + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup("modex"); + kv->value = (pmix_value_t *)malloc(sizeof(pmix_value_t)); + rc = pmix_value_xfer(kv->value, val); + if (PMIX_SUCCESS != (rc = pmix_dstore_store(nptr->nspace, cur_rank, kv))) { + PMIX_ERROR_LOG(rc); + } + PMIX_RELEASE(kv); +#else + pmix_buffer_t xfer, *xptr; pmix_bfrop.pack(&pbkt, &cur_rank, 1, PMIX_INT); /* the client is expecting this to arrive as a byte object * containing a buffer, so package it accordingly */ @@ -420,6 +437,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, int rank, xfer.base_ptr = NULL; // protect the passed data xfer.bytes_used = 0; PMIX_DESTRUCT(&xfer); +#endif /* PMIX_ENABLE_DSTORE */ PMIX_VALUE_RELEASE(val); found++; } diff --git a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_listener.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c similarity index 77% rename from opal/mca/pmix/pmix114/pmix/src/server/pmix_server_listener.c rename to opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c index 2c18aadbe4..1d8e9d57c3 100644 --- a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_listener.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2015 Research Organization for Information Science + * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. @@ -64,7 +64,7 @@ // local functions for connection support static void* listen_thread(void *obj); -static void listener_cb(int incoming_sd); +static void listener_cb(int incoming_sd, void *cbdata); static void connection_handler(int incoming_sd, short flags, void* cbdata); static char *myversion = NULL; static pthread_t engine; @@ -72,50 +72,58 @@ static pthread_t engine; /* * start listening on our rendezvous file */ -pmix_status_t pmix_start_listening(struct sockaddr_un *address, - mode_t mode, uid_t sockuid, gid_t sockgid) +pmix_status_t pmix_start_listening(pmix_listener_t *lt) { int flags; pmix_status_t rc; socklen_t addrlen; char *ptr; + struct sockaddr_un *address = <->address; /* create a listen socket for incoming connection attempts */ - pmix_server_globals.listen_socket = socket(PF_UNIX, SOCK_STREAM, 0); - if (pmix_server_globals.listen_socket < 0) { + lt->socket = socket(PF_UNIX, SOCK_STREAM, 0); + if (lt->socket < 0) { printf("%s:%d socket() failed\n", __FILE__, __LINE__); return PMIX_ERROR; } addrlen = sizeof(struct sockaddr_un); - if (bind(pmix_server_globals.listen_socket, (struct sockaddr*)address, addrlen) < 0) { + if (bind(lt->socket, (struct sockaddr*)address, addrlen) < 0) { printf("%s:%d bind() failed\n", __FILE__, __LINE__); return PMIX_ERROR; } /* chown as required */ - if (0 != chown(address->sun_path, sockuid, sockgid)) { - pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); - goto sockerror; + if (lt->owner_given) { + if (0 != chown(address->sun_path, lt->owner, -1)) { + pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); + goto sockerror; + } + } + if (lt->group_given) { + if (0 != chown(address->sun_path, -1, lt->group)) { + pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); + goto sockerror; + } } /* set the mode as required */ - if (0 != chmod(address->sun_path, mode)) { + if (0 != chmod(address->sun_path, lt->mode)) { pmix_output(0, "CANNOT CHMOD socket %s: %s", address->sun_path, strerror (errno)); goto sockerror; } /* setup listen backlog to maximum allowed by kernel */ - if (listen(pmix_server_globals.listen_socket, SOMAXCONN) < 0) { + if (listen(lt->socket, SOMAXCONN) < 0) { printf("%s:%d listen() failed\n", __FILE__, __LINE__); goto sockerror; } /* set socket up to be non-blocking, otherwise accept could block */ - if ((flags = fcntl(pmix_server_globals.listen_socket, F_GETFL, 0)) < 0) { + if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) { printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__); goto sockerror; } flags |= O_NONBLOCK; - if (fcntl(pmix_server_globals.listen_socket, F_SETFL, flags) < 0) { + if (fcntl(lt->socket, F_SETFL, flags) < 0) { printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__); goto sockerror; } @@ -136,10 +144,10 @@ pmix_status_t pmix_start_listening(struct sockaddr_un *address, /* if the server will listen for us, then ask it to do so now */ rc = PMIX_ERR_NOT_SUPPORTED; if (NULL != pmix_host_server.listener) { - rc = pmix_host_server.listener(pmix_server_globals.listen_socket, listener_cb); + rc = pmix_host_server.listener(lt->socket, listener_cb, (void*)lt); } - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && !pmix_server_globals.listen_thread_active) { /*** spawn internal listener thread */ if (0 > pipe(pmix_server_globals.stop_thread)) { PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO); @@ -165,14 +173,15 @@ pmix_status_t pmix_start_listening(struct sockaddr_un *address, return PMIX_SUCCESS; sockerror: - (void)close(pmix_server_globals.listen_socket); - pmix_server_globals.listen_socket = -1; + (void)close(lt->socket); + lt->socket = -1; return PMIX_ERROR; } void pmix_stop_listening(void) { int i; + pmix_listener_t *lt; pmix_output_verbose(8, pmix_globals.debug_output, "listen_thread: shutdown"); @@ -193,8 +202,11 @@ void pmix_stop_listening(void) } /* wait for thread to exit */ pthread_join(engine, NULL); - /* close the socket to remove the connection point */ - CLOSE_THE_SOCKET(pmix_server_globals.listen_socket); + /* close the sockets to remove the connection points */ + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + CLOSE_THE_SOCKET(lt->socket); + lt->socket = -1; + } return; } @@ -205,14 +217,18 @@ static void* listen_thread(void *obj) pmix_pending_connection_t *pending_connection; struct timeval timeout; fd_set readfds; + pmix_listener_t *lt; pmix_output_verbose(8, pmix_globals.debug_output, "listen_thread: active"); while (pmix_server_globals.listen_thread_active) { FD_ZERO(&readfds); - FD_SET(pmix_server_globals.listen_socket, &readfds); - max = pmix_server_globals.listen_socket; + max = -1; + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + FD_SET(lt->socket, &readfds); + max = (lt->socket > max) ? lt->socket : max; + } /* add the stop_thread fd */ FD_SET(pmix_server_globals.stop_thread[0], &readfds); max = (pmix_server_globals.stop_thread[0] > max) ? pmix_server_globals.stop_thread[0] : max; @@ -241,51 +257,55 @@ static void* listen_thread(void *obj) */ do { accepted_connections = 0; - /* according to the man pages, select replaces the given descriptor - * set with a subset consisting of those descriptors that are ready - * for the specified operation - in this case, a read. So we need to - * first check to see if this file descriptor is included in the - * returned subset - */ - if (0 == FD_ISSET(pmix_server_globals.listen_socket, &readfds)) { - /* this descriptor is not included */ - continue; - } + PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - /* this descriptor is ready to be read, which means a connection - * request has been received - so harvest it. All we want to do - * here is accept the connection and push the info onto the event - * library for subsequent processing - we don't want to actually - * process the connection here as it takes too long, and so the - * OS might start rejecting connections due to timeout. - */ - pending_connection = PMIX_NEW(pmix_pending_connection_t); - event_assign(&pending_connection->ev, pmix_globals.evbase, -1, - EV_WRITE, connection_handler, pending_connection); - pending_connection->sd = accept(pmix_server_globals.listen_socket, - (struct sockaddr*)&(pending_connection->addr), - &addrlen); - if (pending_connection->sd < 0) { - PMIX_RELEASE(pending_connection); - if (pmix_socket_errno != EAGAIN || - pmix_socket_errno != EWOULDBLOCK) { - if (EMFILE == pmix_socket_errno) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - } else { - pmix_output(0, "listen_thread: accept() failed: %s (%d).", - strerror(pmix_socket_errno), pmix_socket_errno); - } - goto done; + /* according to the man pages, select replaces the given descriptor + * set with a subset consisting of those descriptors that are ready + * for the specified operation - in this case, a read. So we need to + * first check to see if this file descriptor is included in the + * returned subset + */ + if (0 == FD_ISSET(lt->socket, &readfds)) { + /* this descriptor is not included */ + continue; } - continue; - } - pmix_output_verbose(8, pmix_globals.debug_output, - "listen_thread: new connection: (%d, %d)", - pending_connection->sd, pmix_socket_errno); - /* activate the event */ - event_active(&pending_connection->ev, EV_WRITE, 1); - accepted_connections++; + /* this descriptor is ready to be read, which means a connection + * request has been received - so harvest it. All we want to do + * here is accept the connection and push the info onto the event + * library for subsequent processing - we don't want to actually + * process the connection here as it takes too long, and so the + * OS might start rejecting connections due to timeout. + */ + pending_connection = PMIX_NEW(pmix_pending_connection_t); + pending_connection->protocol = lt->protocol_type; + event_assign(&pending_connection->ev, pmix_globals.evbase, -1, + EV_WRITE, connection_handler, pending_connection); + pending_connection->sd = accept(lt->socket, + (struct sockaddr*)&(pending_connection->addr), + &addrlen); + if (pending_connection->sd < 0) { + PMIX_RELEASE(pending_connection); + if (pmix_socket_errno != EAGAIN || + pmix_socket_errno != EWOULDBLOCK) { + if (EMFILE == pmix_socket_errno) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + } else { + pmix_output(0, "listen_thread: accept() failed: %s (%d).", + strerror(pmix_socket_errno), pmix_socket_errno); + } + goto done; + } + continue; + } + + pmix_output_verbose(8, pmix_globals.debug_output, + "listen_thread: new connection: (%d, %d)", + pending_connection->sd, pmix_socket_errno); + /* activate the event */ + event_active(&pending_connection->ev, EV_WRITE, 1); + accepted_connections++; + } } while (accepted_connections > 0); } @@ -294,9 +314,10 @@ static void* listen_thread(void *obj) return NULL; } -static void listener_cb(int incoming_sd) +static void listener_cb(int incoming_sd, void *cbdata) { pmix_pending_connection_t *pending_connection; + pmix_listener_t *lt = (pmix_listener_t*)cbdata; /* throw it into our event library for processing */ pmix_output_verbose(8, pmix_globals.debug_output, @@ -304,6 +325,7 @@ static void listener_cb(int incoming_sd) incoming_sd); pending_connection = PMIX_NEW(pmix_pending_connection_t); pending_connection->sd = incoming_sd; + pending_connection->protocol = lt->protocol_type; event_assign(&pending_connection->ev, pmix_globals.evbase, -1, EV_WRITE, connection_handler, pending_connection); event_active(&pending_connection->ev, EV_WRITE, 1); @@ -358,7 +380,8 @@ static pmix_status_t parse_connect_ack (char *msg, int len, /* Receive the peer's identification info from a newly * connected socket and verify the expected response. */ -static pmix_status_t pmix_server_authenticate(int sd, int *out_rank, +static pmix_status_t pmix_server_authenticate(int sd, uint16_t protocol, + int *out_rank, pmix_peer_t **peer) { char *msg, *nspace, *version, *cred; @@ -511,6 +534,16 @@ static pmix_status_t pmix_server_authenticate(int sd, int *out_rank, } } + /* let the host server know that this client has connected */ + if (NULL != pmix_host_server.client_connected) { + (void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN); + proc.rank = psave->info->rank; + rc = pmix_host_server.client_connected(&proc, psave->info->server_object, + NULL, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } /* send the client's array index */ if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(sd, (char*)&psave->index, sizeof(int)))) { PMIX_ERROR_LOG(rc); @@ -523,15 +556,6 @@ static pmix_status_t pmix_server_authenticate(int sd, int *out_rank, "connect-ack from client completed"); *peer = psave; - /* let the host server know that this client has connected */ - if (NULL != pmix_host_server.client_connected) { - (void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN); - proc.rank = psave->info->rank; - rc = pmix_host_server.client_connected(&proc, psave->info->server_object); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - } - } return rc; error: @@ -561,7 +585,8 @@ static void connection_handler(int sd, short flags, void* cbdata) /* receive identifier info from the client and authenticate it - the * function will lookup and return the peer object if the connection * is successfully authenticated */ - if (PMIX_SUCCESS != pmix_server_authenticate(pnd->sd, &rank, &peer)) { + if (PMIX_SUCCESS != pmix_server_authenticate(pnd->sd, pnd->protocol, + &rank, &peer)) { CLOSE_THE_SOCKET(pnd->sd); return; } diff --git a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_ops.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c similarity index 79% rename from opal/mca/pmix/pmix114/pmix/src/server/pmix_server_ops.c rename to opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c index 9d7db586ce..41e46069fd 100644 --- a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_ops.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c @@ -984,16 +984,35 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, { int32_t cnt; pmix_status_t rc; + pmix_status_t *codes = NULL; pmix_info_t *info = NULL; - size_t ninfo, n; + size_t ninfo, ncodes, n, k; pmix_regevents_info_t *reginfo; + pmix_peer_events_info_t *prev; + pmix_notify_caddy_t *cd; + int i; + bool enviro_events = false; + bool found; pmix_output_verbose(2, pmix_globals.debug_output, "recvd register events"); - if (NULL == pmix_host_server.register_events) { - return PMIX_ERR_NOT_SUPPORTED; + /* unpack the number of codes */ + cnt=1; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ncodes, &cnt, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + return rc; } + /* unpack the array of codes */ + if (0 < ncodes) { + codes = (pmix_status_t*)malloc(ncodes * sizeof(pmix_status_t)); + cnt=ncodes; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, codes, &cnt, PMIX_STATUS))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + } + /* unpack the number of info objects */ cnt=1; if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ninfo, &cnt, PMIX_SIZE))) { @@ -1009,138 +1028,311 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, goto cleanup; } } - /* store the event registration info so we can call the registered - client when the server notifies the event */ - reginfo = PMIX_NEW(pmix_regevents_info_t); - if (0 < ninfo) { - PMIX_INFO_CREATE (reginfo->info, ninfo); - reginfo->ninfo = ninfo; - for (n=0; n < ninfo; n++) { - memcpy(reginfo->info[n].key, info[n].key, PMIX_MAX_KEYLEN); - pmix_value_xfer(®info->info[n].value, &info[n].value); - } - } - PMIX_RETAIN(peer); - reginfo->peer = peer; - pmix_list_append(&pmix_server_globals.client_eventregs, ®info->super); - pmix_output_verbose(2, pmix_globals.debug_output, - "server register events: calling host server reg events"); - /* call the local server */ - if(PMIX_SUCCESS != (rc = pmix_host_server.register_events(reginfo->info, - reginfo->ninfo, cbfunc, cbdata))) - { - pmix_output_verbose(2, pmix_globals.debug_output, - "server register events: host server reg events returned rc =%d", rc); - } -cleanup: - pmix_output_verbose(2, pmix_globals.debug_output, - "server register events: ninfo =%lu rc =%d", ninfo, rc); - PMIX_INFO_FREE(info, ninfo); - return rc; -} - -pmix_status_t pmix_server_deregister_events(pmix_peer_t *peer, - pmix_buffer_t *buf, - pmix_op_cbfunc_t cbfunc, - void *cbdata) -{ - int32_t cnt; - pmix_status_t rc; - pmix_info_t *info = NULL; - size_t ninfo; - pmix_regevents_info_t *reginfo = NULL; - pmix_regevents_info_t *reginfo_next; - pmix_output_verbose(2, pmix_globals.debug_output, - "recvd deregister events"); - - if (NULL == pmix_host_server.register_events) { - return PMIX_ERR_NOT_SUPPORTED; - } - /* unpack the number of info objects */ - cnt=1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ninfo, &cnt, PMIX_SIZE))) { - PMIX_ERROR_LOG(rc); - return rc; - } - /* unpack the array of info objects */ - if (0 < ninfo) { - PMIX_INFO_CREATE(info, ninfo); - cnt=ninfo; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, info, &cnt, PMIX_INFO))) { - PMIX_ERROR_LOG(rc); - goto cleanup; - } - } - /* delete the stored event registration info */ - PMIX_LIST_FOREACH_SAFE(reginfo, reginfo_next, - &pmix_server_globals.client_eventregs, pmix_regevents_info_t) { - /* TO DO: For now assume there is one reginfo per peer, we need to revisit this - to match info keys too, inorder to support multiple event reg requests per process */ - if(reginfo->peer == peer) { - pmix_list_remove_item (&pmix_server_globals.client_eventregs, ®info->super); - PMIX_RELEASE(reginfo); + /* see if they asked for enviro events */ + for (n=0; n < ninfo; n++) { + if (0 == strcmp(info[n].key, PMIX_EVENT_ENVIRO_LEVEL)) { + if (PMIX_UNDEF == info[n].value.type || + (PMIX_BOOL == info[n].value.type && info[n].value.data.flag)) { + enviro_events = true; + } break; } } - /* call the local server */ - rc = pmix_host_server.deregister_events(info, ninfo, cbfunc, cbdata); + + /* if they asked for enviro events, and our host doesn't support + * register_events, then we cannot meet the request */ + if (enviro_events && NULL == pmix_host_server.register_events) { + enviro_events = false; + rc = PMIX_ERR_NOT_SUPPORTED; + goto cleanup; + } + + /* store the event registration info so we can call the registered + * client when the server notifies the event */ + k=0; + do { + found = false; + PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { + if (NULL == codes) { + if (PMIX_MAX_ERR_CONSTANT == reginfo->code) { + /* both are default handlers */ + found = true; + break; + } else { + continue; + } + } else { + if (PMIX_MAX_ERR_CONSTANT == reginfo->code) { + continue; + } else if (codes[k] == reginfo->code) { + found = true; + break; + } + } + } + if (found) { + /* found it - add this peer if we don't already have it */ + found = false; + PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { + if (prev->peer == peer) { + /* already have it */ + rc = PMIX_SUCCESS; + found = true; + break; + } + } + if (!found) { + /* get here if we don't already have this peer */ + prev = PMIX_NEW(pmix_peer_events_info_t); + PMIX_RETAIN(peer); + prev->peer = peer; + prev->enviro_events = enviro_events; + pmix_list_append(®info->peers, &prev->super); + found = true; + } + } else { + /* if we get here, then we didn't find an existing registration for this code */ + reginfo = PMIX_NEW(pmix_regevents_info_t); + if (NULL == codes) { + reginfo->code = PMIX_MAX_ERR_CONSTANT; + } else { + reginfo->code = codes[k]; + } + pmix_list_append(&pmix_server_globals.events, ®info->super); + prev = PMIX_NEW(pmix_peer_events_info_t); + PMIX_RETAIN(peer); + prev->peer = peer; + prev->enviro_events = enviro_events; + pmix_list_append(®info->peers, &prev->super); + } + ++k; + } while (k < ncodes); + + /* if they asked for enviro events, call the local server */ + if (enviro_events) { + if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(codes, ncodes, info, ninfo, cbfunc, cbdata))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "server register events: host server reg events returned rc =%d", rc); + } else { + goto check; + } + } cleanup: - PMIX_INFO_FREE(info, ninfo); + pmix_output_verbose(2, pmix_globals.debug_output, + "server register events: ninfo =%lu rc =%d", ninfo, rc); + /* be sure to execute the callback */ + if (NULL != cbfunc) { + cbfunc(rc, cbdata); + } + if (NULL != info) { + PMIX_INFO_FREE(info, ninfo); + } + if (PMIX_SUCCESS != rc) { + if (!enviro_events) { + if (NULL != codes) { + free(codes); + } + } + return rc; + } + + check: + /* check if any matching notifications have been cached */ + for (i=0; i < pmix_server_globals.notifications.size; i++) { + if (NULL == (cd = (pmix_notify_caddy_t*)pmix_ring_buffer_poke(&pmix_server_globals.notifications, i))) { + break; + } + found = false; + if (NULL == codes) { + /* they registered a default event handler - always matches */ + found = true; + } else { + for (k=0; k < ncodes; k++) { + if (codes[k] == cd->status) { + found = true; + break; + } + } + } + if (found) { + /* have a match - notify */ + PMIX_RETAIN(cd->buf); + PMIX_SERVER_QUEUE_REPLY(peer, 0, cd->buf); + } + } + if (!enviro_events) { + if (NULL != codes) { + free(codes); + } + } + + return PMIX_SUCCESS; +} + +pmix_status_t pmix_server_deregister_events(pmix_peer_t *peer, + pmix_buffer_t *buf, + pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + int32_t cnt; + pmix_status_t rc, *codes = NULL, *cdptr, maxcode = PMIX_MAX_ERR_CONSTANT; + pmix_info_t *info = NULL; + size_t ninfo, ncodes, ncds, n; + pmix_regevents_info_t *reginfo = NULL; + pmix_regevents_info_t *reginfo_next; + pmix_peer_events_info_t *prev; + + pmix_output_verbose(2, pmix_globals.debug_output, + "recvd deregister events"); + + /* unpack the number of codes */ + cnt=1; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ncodes, &cnt, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* unpack the array of codes */ + if (0 < ncodes) { + codes = (pmix_status_t*)malloc(ncodes * sizeof(pmix_status_t)); + cnt=ncodes; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, codes, &cnt, PMIX_STATUS))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + } + + /* unpack the number of info objects */ + cnt=1; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ninfo, &cnt, PMIX_SIZE))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* unpack the array of info objects */ + if (0 < ninfo) { + PMIX_INFO_CREATE(info, ninfo); + cnt=ninfo; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, info, &cnt, PMIX_INFO))) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + } + + /* find the event registration info so we can delete them */ + if (NULL == codes) { + cdptr = &maxcode; + ncds = 1; + } else { + cdptr = codes; + ncds = ncodes; + } + + for (n=0; n < ncds; n++) { + PMIX_LIST_FOREACH_SAFE(reginfo, reginfo_next, &pmix_server_globals.events, pmix_regevents_info_t) { + if (cdptr[n] == reginfo->code) { + /* found it - remove this peer from the list */ + PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { + if (prev->peer == peer) { + /* found it */ + pmix_list_remove_item(®info->peers, &prev->super); + PMIX_RELEASE(prev); + break; + } + } + /* if all of the peers for this code are now gone, then remove it */ + if (0 == pmix_list_get_size(®info->peers)) { + pmix_list_remove_item(&pmix_server_globals.events, ®info->super); + /* if this was registered with the host, then deregister it */ + PMIX_RELEASE(reginfo); + } + } + } + } + + +cleanup: + if (NULL != codes) { + free(codes); + } + if (NULL != info) { + PMIX_INFO_FREE(info, ninfo); + } return rc; } -pmix_status_t pmix_server_notify_error_client(pmix_peer_t *peer, - pmix_buffer_t *buf, - pmix_op_cbfunc_t cbfunc, - void *cbdata) + +static void local_cbfunc(pmix_status_t status, void *cbdata) +{ + pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; + + if (NULL != cd->cbfunc) { + cd->cbfunc(status, cd->cbdata); + } + PMIX_RELEASE(cd); +} + +pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, + pmix_buffer_t *buf, + pmix_op_cbfunc_t cbfunc, + void *cbdata) { int32_t cnt; - pmix_status_t rc, status; - pmix_info_t *info = NULL; - size_t ninfo, nprocs; - pmix_proc_t *procs = NULL; + pmix_status_t rc; + pmix_notify_caddy_t *cd; + pmix_output_verbose(2, pmix_globals.debug_output, - "recvd notify error from client"); + "recvd event notification from client"); + + if (NULL == pmix_host_server.notify_event) { + return PMIX_ERR_NOT_SUPPORTED; + } + + cd = PMIX_NEW(pmix_notify_caddy_t); + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + /* set the source */ + (void)strncpy(cd->source.nspace, peer->info->nptr->nspace, PMIX_MAX_NSLEN); + cd->source.rank = peer->info->rank; + /* unpack status */ cnt = 1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &status, &cnt, PMIX_INT))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cd->status, &cnt, PMIX_INT))) { PMIX_ERROR_LOG(rc); goto exit; } - /* unpack procs */ + + /* unpack the range */ cnt = 1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &nprocs, &cnt, PMIX_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cd->range, &cnt, PMIX_DATA_RANGE))) { PMIX_ERROR_LOG(rc); goto exit; } - if ( 0 < nprocs) { - PMIX_PROC_CREATE(procs, nprocs); - cnt = nprocs; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, procs, &cnt, PMIX_PROC))) { - PMIX_ERROR_LOG(rc); - goto exit; - } - } + /* unpack the info keys */ cnt = 1; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &ninfo, &cnt, PMIX_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cd->ninfo, &cnt, PMIX_SIZE))) { PMIX_ERROR_LOG(rc); goto exit; } - if (0 < ninfo) { - PMIX_INFO_CREATE(info, ninfo); - cnt = ninfo; - if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, info, &cnt, PMIX_INFO))) { + if (0 < cd->ninfo) { + PMIX_INFO_CREATE(cd->info, cd->ninfo); + cnt = cd->ninfo; + if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, cd->info, &cnt, PMIX_INFO))) { PMIX_ERROR_LOG(rc); goto exit; } } - pmix_errhandler_invoke(status, procs, nprocs, info, ninfo); -exit: - PMIX_PROC_FREE(procs, nprocs); - PMIX_INFO_FREE(info, ninfo); + + /* when we receive an event from a client, we just pass it to + * our host RM for distribution - if any targeted recipients + * are local to us, the host RM will let us know */ + pmix_host_server.notify_event(cd->status, &cd->source, cd->range, + cd->info, cd->ninfo, local_cbfunc, cd); + return PMIX_SUCCESS; + + exit: + PMIX_RELEASE(cd); cbfunc(rc, cbdata); return rc; } @@ -1218,16 +1410,19 @@ PMIX_CLASS_INSTANCE(pmix_setup_caddy_t, static void ncon(pmix_notify_caddy_t *p) { p->active = true; - p->procs = NULL; - p->nprocs = 0; - p->error_procs = NULL; - p->error_nprocs = 0; + memset(p->source.nspace, 0, PMIX_MAX_NSLEN+1); + p->source.rank = PMIX_RANK_UNDEF; + p->range = PMIX_RANGE_UNDEF; + p->nondefault = false; p->info = NULL; p->ninfo = 0; p->buf = PMIX_NEW(pmix_buffer_t); } static void ndes(pmix_notify_caddy_t *p) { + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } if (NULL != p->buf) { PMIX_RELEASE(p->buf); } @@ -1236,6 +1431,7 @@ PMIX_CLASS_INSTANCE(pmix_notify_caddy_t, pmix_object_t, ncon, ndes); + PMIX_CLASS_INSTANCE(pmix_trkr_caddy_t, pmix_object_t, NULL, NULL); @@ -1277,18 +1473,60 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_local_t, PMIX_CLASS_INSTANCE(pmix_pending_connection_t, pmix_object_t, NULL, NULL); -static void regcon(pmix_regevents_info_t *p) + +static void prevcon(pmix_peer_events_info_t *p) { p->peer = NULL; - p->info = NULL; - p->ninfo = 0; +} +static void prevdes(pmix_peer_events_info_t *p) +{ + if (NULL != p->peer) { + PMIX_RELEASE(p->peer); + } +} +PMIX_CLASS_INSTANCE(pmix_peer_events_info_t, + pmix_list_item_t, + prevcon, prevdes); + +static void regcon(pmix_regevents_info_t *p) +{ + PMIX_CONSTRUCT(&p->peers, pmix_list_t); } static void regdes(pmix_regevents_info_t *p) { - if(NULL != p->peer) - PMIX_RELEASE(p->peer); - PMIX_INFO_FREE(p->info, p->ninfo); + PMIX_LIST_DESTRUCT(&p->peers); } PMIX_CLASS_INSTANCE(pmix_regevents_info_t, pmix_list_item_t, regcon, regdes); + +static void lcon(pmix_listener_t *p) +{ + memset(&p->address, 0, sizeof(struct sockaddr_un)); + p->address.sun_family = AF_UNIX; + p->socket = -1; + p->varname = NULL; + p->uri = NULL; + p->owner_given = false; + p->group_given = false; + p->mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH; +} +static void ldes(pmix_listener_t *p) +{ + if (0 <= p->socket) { + CLOSE_THE_SOCKET(p->socket); + } + if (NULL != p->varname) { + free(p->varname); + } + if (NULL != p->uri) { + free(p->uri); + } +} +PMIX_CLASS_INSTANCE(pmix_listener_t, + pmix_list_item_t, + lcon, ldes); + +PMIX_CLASS_INSTANCE(pmix_usock_queue_t, + pmix_object_t, + NULL, NULL); diff --git a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_ops.h b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h similarity index 71% rename from opal/mca/pmix/pmix114/pmix/src/server/pmix_server_ops.h rename to opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h index 1b946441bc..336c0beaea 100644 --- a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_ops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h @@ -15,6 +15,7 @@ #include #include +#include #include #include "src/usock/usock.h" #include "src/util/hash.h" @@ -48,10 +49,9 @@ typedef struct { pmix_event_t ev; volatile bool active; pmix_status_t status; - pmix_proc_t *procs; - size_t nprocs; - pmix_proc_t *error_procs; - size_t error_nprocs; + pmix_proc_t source; + pmix_data_range_t range; + bool nondefault; pmix_info_t *info; size_t ninfo; pmix_buffer_t *buf; @@ -87,11 +87,9 @@ PMIX_CLASS_DECLARATION(pmix_dmdx_local_t); typedef struct { pmix_object_t super; pmix_event_t ev; - char nspace[PMIX_MAX_NSLEN+1]; - pmix_status_t status; + uint16_t protocol; int sd; struct sockaddr_storage addr; - char *msg; } pmix_pending_connection_t; PMIX_CLASS_DECLARATION(pmix_pending_connection_t); @@ -99,23 +97,55 @@ PMIX_CLASS_DECLARATION(pmix_pending_connection_t); typedef struct { pmix_list_item_t super; pmix_peer_t *peer; - pmix_info_t *info; - size_t ninfo; + bool enviro_events; +} pmix_peer_events_info_t; +PMIX_CLASS_DECLARATION(pmix_peer_events_info_t); + +typedef struct { + pmix_list_item_t super; + pmix_list_t peers; // list of pmix_prevents_info_t + int code; } pmix_regevents_info_t; PMIX_CLASS_DECLARATION(pmix_regevents_info_t); +/* listener objects */ +typedef struct pmix_listener_t { + pmix_list_item_t super; + uint16_t protocol_type; + int socket; + struct sockaddr_un address; + char *varname; + char *uri; + uint32_t owner; + bool owner_given; + uint32_t group; + bool group_given; + uint32_t mode; +} pmix_listener_t; +PMIX_CLASS_DECLARATION(pmix_listener_t); + typedef struct { pmix_pointer_array_t clients; // array of pmix_peer_t local clients pmix_list_t collectives; // list of active pmix_server_trkr_t pmix_list_t remote_pnd; // list of pmix_dmdx_remote_t awaiting arrival of data fror servicing remote req's pmix_list_t local_reqs; // list of pmix_dmdx_local_t awaiting arrival of data from local neighbours volatile bool listen_thread_active; // listen thread is running - int listen_socket; // socket listener is watching + pmix_list_t listeners; // list of pmix_listener_t int stop_thread[2]; // pipe used to stop listener thread pmix_buffer_t gdata; // cache of data given to me for passing to all clients - pmix_list_t client_eventregs; // list of registered events per client. + pmix_list_t events; // list of pmix_regevents_info_t registered events + pmix_ring_buffer_t notifications; // ring buffer of pending notifications } pmix_server_globals_t; +typedef struct { + pmix_object_t super; + pmix_event_t ev; + pmix_peer_t *peer; + pmix_buffer_t *buf; + uint32_t tag; +} pmix_usock_queue_t; +PMIX_CLASS_DECLARATION(pmix_usock_queue_t); + #define PMIX_PEER_CADDY(c, p, t) \ do { \ (c) = PMIX_NEW(pmix_server_caddy_t); \ @@ -147,8 +177,31 @@ typedef struct { } while (0) -pmix_status_t pmix_start_listening(struct sockaddr_un *address, - mode_t mode, uid_t sockuid, gid_t sockgid); +/* queue a message to be sent to one of our procs - must + * provide the following params: + * + * t - tag to be sent to + * b - buffer to be sent + */ +#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \ + do { \ + pmix_usock_queue_t *queue; \ + queue = PMIX_NEW(pmix_usock_queue_t); \ + queue->peer = (p); \ + queue->buf = (b); \ + queue->tag = (t); \ + pmix_output_verbose(2, pmix_globals.debug_output, \ + "[%s:%d] queue reply to %s:%d on tag %d", \ + __FILE__, __LINE__, \ + (queue->peer)->info->nptr->nspace, \ + (queue->peer)->info->rank, (queue->tag)); \ + event_assign(&queue->ev, pmix_globals.evbase, -1, \ + EV_WRITE, pmix_server_queue_message, queue); \ + event_priority_set(&queue->ev, 0); \ + event_active(&queue->ev, EV_WRITE, 1); \ + } while (0) + +pmix_status_t pmix_start_listening(pmix_listener_t *lt); void pmix_stop_listening(void); bool pmix_server_trk_update(pmix_server_trkr_t *trk); @@ -201,15 +254,6 @@ void pmix_pack_proc_map(pmix_buffer_t *buf, pmix_status_t pmix_regex_parse_nodes(const char *regexp, char ***names); pmix_status_t pmix_regex_parse_procs(const char *regexp, char ***procs); -void pmix_server_register_errhandler(pmix_info_t info[], size_t ninfo, - pmix_notification_fn_t errhandler, - pmix_errhandler_reg_cbfunc_t cbfunc, - void *cbdata); - -void pmix_server_deregister_errhandler(int errhandler_ref, - pmix_op_cbfunc_t cbfunc, - void *cbdata); - pmix_status_t pmix_server_notify_error(pmix_status_t status, pmix_proc_t procs[], size_t nprocs, pmix_proc_t error_procs[], size_t error_nprocs, @@ -217,26 +261,23 @@ pmix_status_t pmix_server_notify_error(pmix_status_t status, pmix_op_cbfunc_t cbfunc, void *cbdata); pmix_status_t pmix_server_register_events(pmix_peer_t *peer, - pmix_buffer_t *buf, - pmix_op_cbfunc_t cbfunc, - void *cbdata); - -pmix_status_t pmix_server_deregister_events(pmix_peer_t *peer, pmix_buffer_t *buf, pmix_op_cbfunc_t cbfunc, void *cbdata); -pmix_status_t pmix_server_notify_error_client(pmix_peer_t *peer, - pmix_buffer_t *buf, - pmix_op_cbfunc_t cbfunc, - void *cbdata); -void pmix_server_check_notifications(pmix_regevents_info_t *reginfo, - pmix_notify_caddy_t *cd); - -void regevents_cbfunc (pmix_status_t status, void *cbdata); +pmix_status_t pmix_server_deregister_events(pmix_peer_t *peer, + pmix_buffer_t *buf, + pmix_op_cbfunc_t cbfunc, + void *cbdata); +pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, + pmix_buffer_t *buf, + pmix_op_cbfunc_t cbfunc, + void *cbdata); void pmix_server_execute_collective(int sd, short args, void *cbdata); +void pmix_server_queue_message(int fd, short args, void *cbdata); + extern pmix_server_module_t pmix_host_server; extern pmix_server_globals_t pmix_server_globals; diff --git a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_regex.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_regex.c similarity index 99% rename from opal/mca/pmix/pmix114/pmix/src/server/pmix_server_regex.c rename to opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_regex.c index a27824fd52..cc0caa01d4 100644 --- a/opal/mca/pmix/pmix114/pmix/src/server/pmix_server_regex.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_regex.c @@ -529,6 +529,8 @@ static pmix_status_t pmix_regex_extract_ppn(char *regexp, char ***procs) end = strtol(t, NULL, 10); for (k=start; k <= end; k++) { if (0 > asprintf(&t, "%d", k)) { + pmix_argv_free(nds); + pmix_argv_free(rngs); return PMIX_ERR_NOMEM; } pmix_argv_append_nosize(&ps, t); diff --git a/opal/mca/pmix/pmix2x/pmix/src/sm/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/sm/Makefile.am new file mode 100644 index 0000000000..476011224e --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/sm/Makefile.am @@ -0,0 +1,17 @@ +# +# Copyright (c) 2016 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers += \ + src/sm/pmix_sm.h \ + src/sm/pmix_mmap.h + +sources += \ + src/sm/pmix_sm.c \ + src/sm/pmix_mmap.c diff --git a/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c new file mode 100644 index 0000000000..5a104d757f --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2015-2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include +#include +#include + +#include +#include +#include "src/include/pmix_globals.h" + +#include "pmix_sm.h" +#include "pmix_mmap.h" + +#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +# define MAP_ANONYMOUS MAP_ANON +#endif /* MAP_ANONYMOUS and MAP_ANON */ + + +static int _mmap_segment_create(pmix_sm_seg_t *sm_seg, const char *file_name, size_t size); +static int _mmap_segment_attach(pmix_sm_seg_t *sm_seg); +static int _mmap_segment_detach(pmix_sm_seg_t *sm_seg); +static int _mmap_segment_unlink(pmix_sm_seg_t *sm_seg); + +pmix_sm_base_module_t pmix_sm_mmap_module = { + "mmap", + _mmap_segment_create, + _mmap_segment_attach, + _mmap_segment_detach, + _mmap_segment_unlink +}; + + +int _mmap_segment_create(pmix_sm_seg_t *sm_seg, const char *file_name, size_t size) +{ + int rc = PMIX_SUCCESS; + void *seg_addr = MAP_FAILED; + pid_t my_pid = getpid(); + + _segment_ds_reset(sm_seg); + /* enough space is available, so create the segment */ + if (-1 == (sm_seg->seg_id = open(file_name, O_CREAT | O_RDWR, 0600))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call open(2) fail\n"); + rc = PMIX_ERROR; + goto out; + } + /* size backing file - note the use of real_size here */ + if (0 != ftruncate(sm_seg->seg_id, size)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call ftruncate(2) fail\n"); + rc = PMIX_ERROR; + goto out; + } + if (MAP_FAILED == (seg_addr = mmap(NULL, size, + PROT_READ | PROT_WRITE, MAP_SHARED, + sm_seg->seg_id, 0))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call mmap(2) fail\n"); + rc = PMIX_ERROR; + goto out; + } + sm_seg->seg_cpid = my_pid; + sm_seg->seg_size = size; + sm_seg->seg_base_addr = (unsigned char *)seg_addr; + (void)strncpy(sm_seg->seg_name, file_name, PMIX_PATH_MAX - 1); + +out: + if (-1 != sm_seg->seg_id) { + if (0 != close(sm_seg->seg_id)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call close(2) fail\n"); + rc = PMIX_ERROR; + } + } + /* an error occured, so invalidate the shmem object and munmap if needed */ + if (PMIX_SUCCESS != rc) { + if (MAP_FAILED != seg_addr) { + munmap((void *)seg_addr, size); + } + _segment_ds_reset(sm_seg); + } + return rc; +} + +int _mmap_segment_attach(pmix_sm_seg_t *sm_seg) +{ + if (-1 == (sm_seg->seg_id = open(sm_seg->seg_name, O_RDWR))) { + return PMIX_ERROR; + } + if (MAP_FAILED == (sm_seg->seg_base_addr = (unsigned char *) + mmap(NULL, sm_seg->seg_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + sm_seg->seg_id, 0))) { + /* mmap failed, so close the file and return NULL - no error check + * here because we are already in an error path... + */ + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call mmap(2) fail\n"); + close(sm_seg->seg_id); + return PMIX_ERROR; + } + /* all is well */ + /* if close fails here, that's okay. just let the user know and + * continue. if we got this far, open and mmap were successful... + */ + if (0 != close(sm_seg->seg_id)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call close(2) fail\n"); + } + sm_seg->seg_cpid = 0;/* FIXME */ + return PMIX_SUCCESS; +} + +int _mmap_segment_detach(pmix_sm_seg_t *sm_seg) +{ + int rc = PMIX_SUCCESS; + + if (0 != munmap((void *)sm_seg->seg_base_addr, sm_seg->seg_size)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call munmap(2) fail\n"); + rc = PMIX_ERROR; + } + /* reset the contents of the pmix_sm_seg_t associated with this + * shared memory segment. + */ + _segment_ds_reset(sm_seg); + return rc; +} + +int _mmap_segment_unlink(pmix_sm_seg_t *sm_seg) +{ + if (-1 == unlink(sm_seg->seg_name)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call unlink(2) fail\n"); + return PMIX_ERROR; + } + + sm_seg->seg_id = PMIX_SHMEM_DS_ID_INVALID; + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.h b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.h new file mode 100644 index 0000000000..349fb10c01 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2015-2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_SM_MMAP_H +#define PMIX_SM_MMAP_H + +#include + + +#include "pmix_sm.h" + +BEGIN_C_DECLS + +extern pmix_sm_base_module_t pmix_sm_mmap_module; + +END_C_DECLS + +#endif /* PMIX_SM_MMAP_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_sm.c b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_sm.c new file mode 100644 index 0000000000..e508b60550 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_sm.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2015-2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include "src/include/pmix_globals.h" + +#include "pmix_sm.h" +#include "pmix_mmap.h" + + +/* + * Array of all possible SMs + */ + +/**** ENSURE THE FOLLOWING VALUE IS AT LEAST AS + **** LARGE AS THE TOTAL NUMBER OF SUPPORTED SPCs + **** IN THE ARRAY BELOW + */ + +static pmix_sm_base_module_t *all[] = { + &pmix_sm_mmap_module, + + /* Always end the array with a NULL */ + NULL +}; + +pmix_sm_base_module_t pmix_sm = {0}; + +int pmix_sm_init(void) +{ + pmix_sm = *all[0]; + return PMIX_SUCCESS; +} + +void pmix_sm_finalize(void) +{ + return ; +} + +int pmix_sm_segment_create(pmix_sm_seg_t *sm_seg, const char *file_name, size_t size) +{ + if (!pmix_sm.segment_create) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_sm.segment_create(sm_seg, file_name, size); +} + +int pmix_sm_segment_attach(pmix_sm_seg_t *sm_seg) +{ + if (!pmix_sm.segment_attach) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_sm.segment_attach(sm_seg); +} + +int pmix_sm_segment_detach(pmix_sm_seg_t *sm_seg) +{ + if (!pmix_sm.segment_detach) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_sm.segment_detach(sm_seg); +} + +int pmix_sm_segment_unlink(pmix_sm_seg_t *sm_seg) +{ + if (!pmix_sm.segment_unlink) { + return PMIX_ERR_NOT_SUPPORTED; + } + + return pmix_sm.segment_unlink(sm_seg); +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_sm.h b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_sm.h new file mode 100644 index 0000000000..6429b07510 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_sm.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015-2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_SM_H +#define PMIX_SM_H + +#include + + + +BEGIN_C_DECLS + +#if !defined(MAP_FAILED) +# define MAP_FAILED ((char*)-1) +#endif /* MAP_FAILED */ + +#define PMIX_SHMEM_DS_ID_INVALID -1 + + +typedef struct pmix_sm_seg_t { + /* pid of the shared memory segment creator */ + pid_t seg_cpid; + /* ds id */ + int seg_id; + /* size of shared memory segment */ + size_t seg_size; + /* base address of shared memory segment */ + unsigned char *seg_base_addr; + char seg_name[PMIX_PATH_MAX]; +} pmix_sm_seg_t; + +int pmix_sm_init(void); +void pmix_sm_finalize(void); +int pmix_sm_segment_create(pmix_sm_seg_t *sm_seg, const char *file_name, size_t size); +int pmix_sm_segment_attach(pmix_sm_seg_t *sm_seg); +int pmix_sm_segment_detach(pmix_sm_seg_t *sm_seg); +int pmix_sm_segment_unlink(pmix_sm_seg_t *sm_seg); + +static inline void _segment_ds_reset(pmix_sm_seg_t *sm_seg) +{ + sm_seg->seg_cpid = 0; + sm_seg->seg_id = PMIX_SHMEM_DS_ID_INVALID; + sm_seg->seg_size = 0; + memset(sm_seg->seg_name, '\0', PMIX_PATH_MAX); + sm_seg->seg_base_addr = (unsigned char *)MAP_FAILED; +} + + +/** +* create a new shared memory segment and initialize members in structure +* pointed to by sm_seg. +* +* @param sm_seg pointer to pmix_sm_seg_t structure +* +* @param file_name unique string identifier that must be a valid, +* writable path (IN). +* +* @param size size of the shared memory segment. +* +* @return PMIX_SUCCESS on success. +*/ +typedef int (*pmix_sm_base_module_segment_create_fn_t)(pmix_sm_seg_t *sm_seg, const char *file_name, size_t size); + +/** +* attach to an existing shared memory segment initialized by segment_create. +* +* @param sm_seg pointer to initialized pmix_sm_seg_t typedef'd +* structure (IN/OUT). +* +* @return base address of shared memory segment on success. returns +* NULL otherwise. +*/ +typedef int (*pmix_sm_base_module_segment_attach_fn_t)(pmix_sm_seg_t *sm_seg); + +/** +* detach from an existing shared memory segment. +* +* @param sm_seg pointer to initialized pmix_sm_seg_t typedef'd structure +* (IN/OUT). +* +* @return PMIX_SUCCESS on success. +*/ +typedef int (*pmix_sm_base_module_segment_detach_fn_t)(pmix_sm_seg_t *sm_seg); + +/** +* unlink an existing shared memory segment. +* +* @param sm_seg pointer to initialized pmix_sm_seg_t typedef'd structure +* (IN/OUT). +* +* @return PMIX_SUCCESS on success. +*/ +typedef int (*pmix_sm_base_module_unlink_fn_t)(pmix_sm_seg_t *sm_seg); + + +/** +* structure for sm modules +*/ +typedef struct { + const char *name; + pmix_sm_base_module_segment_create_fn_t segment_create; + pmix_sm_base_module_segment_attach_fn_t segment_attach; + pmix_sm_base_module_segment_detach_fn_t segment_detach; + pmix_sm_base_module_unlink_fn_t segment_unlink; +} pmix_sm_base_module_t; + + +END_C_DECLS + +#endif /* PMIX_SM_H */ diff --git a/opal/mca/pmix/pmix114/pmix/src/usock/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/usock/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/src/usock/usock.c b/opal/mca/pmix/pmix2x/pmix/src/usock/usock.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/usock/usock.c rename to opal/mca/pmix/pmix2x/pmix/src/usock/usock.c diff --git a/opal/mca/pmix/pmix114/pmix/src/usock/usock.h b/opal/mca/pmix/pmix2x/pmix/src/usock/usock.h similarity index 99% rename from opal/mca/pmix/pmix114/pmix/src/usock/usock.h rename to opal/mca/pmix/pmix2x/pmix/src/usock/usock.h index 6df1e8feae..c0eb139164 100644 --- a/opal/mca/pmix/pmix114/pmix/src/usock/usock.h +++ b/opal/mca/pmix/pmix2x/pmix/src/usock/usock.h @@ -62,6 +62,8 @@ #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" + + /* usock structure for tracking posted recvs */ typedef struct { pmix_list_item_t super; @@ -98,8 +100,8 @@ typedef struct { pmix_value_cbfunc_t value_cbfunc; pmix_lookup_cbfunc_t lookup_cbfunc; pmix_spawn_cbfunc_t spawn_cbfunc; - pmix_errhandler_reg_cbfunc_t errreg_cbfunc; - int errhandler_ref; + pmix_evhdlr_reg_cbfunc_t errreg_cbfunc; + size_t errhandler_ref; void *cbdata; char nspace[PMIX_MAX_NSLEN+1]; int rank; diff --git a/opal/mca/pmix/pmix114/pmix/src/usock/usock_sendrecv.c b/opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c similarity index 98% rename from opal/mca/pmix/pmix114/pmix/src/usock/usock_sendrecv.c rename to opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c index 18d7a0749f..9f66dc8f2a 100644 --- a/opal/mca/pmix/pmix114/pmix/src/usock/usock_sendrecv.c +++ b/opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c @@ -68,6 +68,7 @@ static void lost_connection(pmix_peer_t *peer, pmix_status_t err) peer->recv_msg = NULL; } CLOSE_THE_SOCKET(peer->sd); + if (pmix_globals.server) { /* if I am a server, then we need to ensure that * we properly account for the loss of this client @@ -101,9 +102,10 @@ static void lost_connection(pmix_peer_t *peer, pmix_status_t err) } /* remove this proc from the list of ranks for this nspace */ pmix_list_remove_item(&(peer->info->nptr->server->ranks), &(peer->info->super)); - PMIX_RELEASE(peer->info); /* reduce the number of local procs */ --peer->info->nptr->server->nlocalprocs; + /* now decrease the refcount - might actually free the object */ + PMIX_RELEASE(peer->info); /* do some cleanup as the client has left us */ pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); @@ -112,8 +114,10 @@ static void lost_connection(pmix_peer_t *peer, pmix_status_t err) /* if I am a client, there is only * one connection we can have */ pmix_globals.connected = false; + /* set the public error status */ + err = PMIX_ERR_LOST_CONNECTION_TO_SERVER; } - PMIX_REPORT_ERROR(err); + PMIX_REPORT_EVENT(err); } static pmix_status_t send_bytes(int sd, char **buf, size_t *remain) @@ -145,7 +149,7 @@ static pmix_status_t send_bytes(int sd, char **buf, size_t *remain) pmix_output(0, "pmix_usock_msg_send_bytes: write failed: %s (%d) [sd = %d]", strerror(pmix_socket_errno), pmix_socket_errno, sd); - ret = PMIX_ERR_COMM_FAILURE; + ret = PMIX_ERR_UNREACH; goto exit; } /* update location */ @@ -530,5 +534,5 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata) /* we get here if no matching recv was found - this is an error */ pmix_output(0, "UNEXPECTED MESSAGE tag =%d", msg->hdr.tag); PMIX_RELEASE(msg); - PMIX_REPORT_ERROR(PMIX_ERROR); + PMIX_REPORT_EVENT(PMIX_ERROR); } diff --git a/opal/mca/pmix/pmix114/pmix/src/util/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/src/util/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/src/util/argv.c b/opal/mca/pmix/pmix2x/pmix/src/util/argv.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/argv.c rename to opal/mca/pmix/pmix2x/pmix/src/util/argv.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/argv.h b/opal/mca/pmix/pmix2x/pmix/src/util/argv.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/argv.h rename to opal/mca/pmix/pmix2x/pmix/src/util/argv.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/basename.c b/opal/mca/pmix/pmix2x/pmix/src/util/basename.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/basename.c rename to opal/mca/pmix/pmix2x/pmix/src/util/basename.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/basename.h b/opal/mca/pmix/pmix2x/pmix/src/util/basename.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/basename.h rename to opal/mca/pmix/pmix2x/pmix/src/util/basename.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/crc.c b/opal/mca/pmix/pmix2x/pmix/src/util/crc.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/crc.c rename to opal/mca/pmix/pmix2x/pmix/src/util/crc.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/crc.h b/opal/mca/pmix/pmix2x/pmix/src/util/crc.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/crc.h rename to opal/mca/pmix/pmix2x/pmix/src/util/crc.h diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/error.c b/opal/mca/pmix/pmix2x/pmix/src/util/error.c new file mode 100644 index 0000000000..acfee9691b --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/error.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007-2012 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + + +#ifdef HAVE_STRING_H +#include +#endif +#include +#include +#ifdef HAVE_STDLIB_H +#include +#endif + +#include + +#include "src/util/error.h" +#include "src/include/pmix_globals.h" +#include "src/buffer_ops/buffer_ops.h" + +const char* PMIx_Error_string(pmix_status_t errnum) +{ + switch(errnum) { + case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: + return "UNPACK-PAST-END"; + case PMIX_ERR_LOST_CONNECTION_TO_SERVER: + return "LOST_CONNECTION_TO_SERVER"; + case PMIX_ERR_NOT_SUPPORTED: + return "NOT-SUPPORTED"; + case PMIX_ERR_NOT_FOUND: + return "NOT-FOUND"; + case PMIX_ERR_SERVER_NOT_AVAIL: + return "SERVER-NOT-AVAIL"; + case PMIX_ERR_INVALID_NAMESPACE: + return "INVALID-NAMESPACE"; + case PMIX_ERR_INVALID_SIZE: + return "INVALID-SIZE"; + case PMIX_ERR_INVALID_KEYVALP: + return "INVALID-KEYVAL"; + case PMIX_ERR_INVALID_NUM_PARSED: + return "INVALID-NUM-PARSED"; + + case PMIX_ERR_INVALID_ARGS: + return "INVALID-ARGS"; + case PMIX_ERR_INVALID_NUM_ARGS: + return "INVALID-NUM-ARGS"; + case PMIX_ERR_INVALID_LENGTH: + return "INVALID-LENGTH"; + case PMIX_ERR_INVALID_VAL_LENGTH: + return "INVALID-VAL-LENGTH"; + case PMIX_ERR_INVALID_VAL: + return "INVALID-VAL"; + case PMIX_ERR_INVALID_KEY_LENGTH: + return "INVALID-KEY-LENGTH"; + case PMIX_ERR_INVALID_KEY: + return "INVALID-KEY"; + case PMIX_ERR_INVALID_ARG: + return "INVALID-ARG"; + case PMIX_ERR_NOMEM: + return "NO-MEM"; + case PMIX_ERR_INIT: + return "INIT"; + + case PMIX_ERR_DATA_VALUE_NOT_FOUND: + return "DATA-VALUE-NOT-FOUND"; + case PMIX_ERR_OUT_OF_RESOURCE: + return "OUT-OF-RESOURCE"; + case PMIX_ERR_BAD_PARAM: + return "BAD-PARAM"; + case PMIX_ERR_IN_ERRNO: + return "ERR-IN-ERRNO"; + case PMIX_ERR_UNREACH: + return "UNREACHABLE"; + case PMIX_ERR_TIMEOUT: + return "TIMEOUT"; + case PMIX_ERR_NO_PERMISSIONS: + return "NO-PERMISSIONS"; + case PMIX_ERR_PACK_MISMATCH: + return "PACK-MISMATCH"; + case PMIX_ERR_PACK_FAILURE: + return "PACK-FAILURE"; + + case PMIX_ERR_UNPACK_FAILURE: + return "UNPACK-FAILURE"; + case PMIX_ERR_UNPACK_INADEQUATE_SPACE: + return "UNPACK-INADEQUATE-SPACE"; + case PMIX_ERR_TYPE_MISMATCH: + return "TYPE-MISMATCH"; + case PMIX_ERR_PROC_ENTRY_NOT_FOUND: + return "PROC-ENTRY-NOT-FOUND"; + case PMIX_ERR_UNKNOWN_DATA_TYPE: + return "UNKNOWN-DATA-TYPE"; + case PMIX_ERR_WOULD_BLOCK: + return "WOULD-BLOCK"; + case PMIX_ERR_READY_FOR_HANDSHAKE: + return "READY-FOR-HANDSHAKE"; + case PMIX_ERR_HANDSHAKE_FAILED: + return "HANDSHAKE-FAILED"; + case PMIX_ERR_INVALID_CRED: + return "INVALID-CREDENTIAL"; + case PMIX_EXISTS: + return "EXISTS"; + case PMIX_ERR_SERVER_FAILED_REQUEST: + return "SERVER FAILED REQUEST"; + case PMIX_ERR_PROC_MIGRATE: + return "PROC-MIGRATE"; + case PMIX_ERR_PROC_CHECKPOINT: + return "PROC-CHECKPOINT-ERROR"; + case PMIX_ERR_PROC_RESTART: + return "PROC_RESTART"; + case PMIX_ERR_PROC_ABORTING: + return "PROC-ABORTING"; + case PMIX_ERR_PROC_REQUESTED_ABORT: + return "PROC-ABORT-REQUESTED"; + case PMIX_ERR_PROC_ABORTED: + return "PROC-ABORTED"; + case PMIX_ERR_DEBUGGER_RELEASE: + return "DEBUGGER-RELEASE"; + case PMIX_ERR_SILENT: + return "SILENT_ERROR"; + case PMIX_ERROR: + return "ERROR"; + case PMIX_SUCCESS: + return "SUCCESS"; + + default: + return "ERROR STRING NOT FOUND"; + } +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/error.h b/opal/mca/pmix/pmix2x/pmix/src/util/error.h new file mode 100644 index 0000000000..b49a0d7ace --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/error.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_UTIL_ERROR_H +#define PMIX_UTIL_ERROR_H + +#include + + +#include +#include "src/util/output.h" + + BEGIN_C_DECLS + +/* internal error codes - never exposed outside of the library */ +#define PMIX_ERR_INVALID_CRED (PMIX_INTERNAL_ERR_BASE - 1) +#define PMIX_ERR_HANDSHAKE_FAILED (PMIX_INTERNAL_ERR_BASE - 2) +#define PMIX_ERR_READY_FOR_HANDSHAKE (PMIX_INTERNAL_ERR_BASE - 3) +#define PMIX_ERR_UNKNOWN_DATA_TYPE (PMIX_INTERNAL_ERR_BASE - 4) +#define PMIX_ERR_TYPE_MISMATCH (PMIX_INTERNAL_ERR_BASE - 5) +#define PMIX_ERR_UNPACK_INADEQUATE_SPACE (PMIX_INTERNAL_ERR_BASE - 6) +#define PMIX_ERR_UNPACK_FAILURE (PMIX_INTERNAL_ERR_BASE - 7) +#define PMIX_ERR_PACK_FAILURE (PMIX_INTERNAL_ERR_BASE - 8) +#define PMIX_ERR_PACK_MISMATCH (PMIX_INTERNAL_ERR_BASE - 9) +#define PMIX_ERR_PROC_ENTRY_NOT_FOUND (PMIX_INTERNAL_ERR_BASE - 10) +#define PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER (PMIX_INTERNAL_ERR_BASE - 11) +#define PMIX_ERR_SERVER_NOT_AVAIL (PMIX_INTERNAL_ERR_BASE - 12) +#define PMIX_ERR_INVALID_KEYVALP (PMIX_INTERNAL_ERR_BASE - 13) +#define PMIX_ERR_INVALID_NUM_PARSED (PMIX_INTERNAL_ERR_BASE - 14) +#define PMIX_ERR_INVALID_ARGS (PMIX_INTERNAL_ERR_BASE - 15) +#define PMIX_ERR_INVALID_NUM_ARGS (PMIX_INTERNAL_ERR_BASE - 16) +#define PMIX_ERR_INVALID_LENGTH (PMIX_INTERNAL_ERR_BASE - 17) +#define PMIX_ERR_INVALID_VAL_LENGTH (PMIX_INTERNAL_ERR_BASE - 18) +#define PMIX_ERR_INVALID_VAL (PMIX_INTERNAL_ERR_BASE - 19) +#define PMIX_ERR_INVALID_KEY_LENGTH (PMIX_INTERNAL_ERR_BASE - 20) +#define PMIX_ERR_INVALID_KEY (PMIX_INTERNAL_ERR_BASE - 21) +#define PMIX_ERR_INVALID_ARG (PMIX_INTERNAL_ERR_BASE - 22) +#define PMIX_ERR_NOMEM (PMIX_INTERNAL_ERR_BASE - 23) +#define PMIX_ERR_IN_ERRNO (PMIX_INTERNAL_ERR_BASE - 24) +#define PMIX_ERR_SILENT (PMIX_INTERNAL_ERR_BASE - 25) +#define PMIX_ERR_UNKNOWN_DATATYPE (PMIX_INTERNAL_ERR_BASE - 26) +#define PMIX_ERR_RESOURCE_BUSY (PMIX_INTERNAL_ERR_BASE - 27) + +#define PMIX_ERROR_LOG(r) \ + do { \ + if (PMIX_ERR_SILENT != (r)) { \ + pmix_output(0, "PMIX ERROR: %s in file %s at line %d", \ + PMIx_Error_string((r)), __FILE__, __LINE__); \ + } \ +} while (0) + + END_C_DECLS + +#endif /* PMIX_UTIL_ERROR_H */ diff --git a/opal/mca/pmix/pmix114/pmix/src/util/fd.c b/opal/mca/pmix/pmix2x/pmix/src/util/fd.c similarity index 95% rename from opal/mca/pmix/pmix114/pmix/src/util/fd.c rename to opal/mca/pmix/pmix2x/pmix/src/util/fd.c index 9b1fb45d69..20f298f4b2 100644 --- a/opal/mca/pmix/pmix114/pmix/src/util/fd.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/fd.c @@ -1,8 +1,7 @@ /* * Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2009 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. - * + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -22,6 +21,7 @@ #include #include +#include "src/util/error.h" #include "src/util/fd.h" diff --git a/opal/mca/pmix/pmix114/pmix/src/util/fd.h b/opal/mca/pmix/pmix2x/pmix/src/util/fd.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/fd.h rename to opal/mca/pmix/pmix2x/pmix/src/util/fd.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/hash.c b/opal/mca/pmix/pmix2x/pmix/src/util/hash.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/hash.c rename to opal/mca/pmix/pmix2x/pmix/src/util/hash.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/hash.h b/opal/mca/pmix/pmix2x/pmix/src/util/hash.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/hash.h rename to opal/mca/pmix/pmix2x/pmix/src/util/hash.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/os_path.c b/opal/mca/pmix/pmix2x/pmix/src/util/os_path.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/os_path.c rename to opal/mca/pmix/pmix2x/pmix/src/util/os_path.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/os_path.h b/opal/mca/pmix/pmix2x/pmix/src/util/os_path.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/os_path.h rename to opal/mca/pmix/pmix2x/pmix/src/util/os_path.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/output.c b/opal/mca/pmix/pmix2x/pmix/src/util/output.c similarity index 99% rename from opal/mca/pmix/pmix114/pmix/src/util/output.c rename to opal/mca/pmix/pmix2x/pmix/src/util/output.c index ffbcfbc4c7..3218b62caf 100644 --- a/opal/mca/pmix/pmix114/pmix/src/util/output.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/output.c @@ -41,6 +41,7 @@ #endif #include "src/util/pmix_environ.h" +#include "src/util/error.h" #include "src/util/output.h" /* diff --git a/opal/mca/pmix/pmix114/pmix/src/util/output.h b/opal/mca/pmix/pmix2x/pmix/src/util/output.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/output.h rename to opal/mca/pmix/pmix2x/pmix/src/util/output.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/pmix_environ.c b/opal/mca/pmix/pmix2x/pmix/src/util/pmix_environ.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/pmix_environ.c rename to opal/mca/pmix/pmix2x/pmix/src/util/pmix_environ.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/pmix_environ.h b/opal/mca/pmix/pmix2x/pmix/src/util/pmix_environ.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/pmix_environ.h rename to opal/mca/pmix/pmix2x/pmix/src/util/pmix_environ.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/printf.c b/opal/mca/pmix/pmix2x/pmix/src/util/printf.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/printf.c rename to opal/mca/pmix/pmix2x/pmix/src/util/printf.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/printf.h b/opal/mca/pmix/pmix2x/pmix/src/util/printf.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/printf.h rename to opal/mca/pmix/pmix2x/pmix/src/util/printf.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/progress_threads.c b/opal/mca/pmix/pmix2x/pmix/src/util/progress_threads.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/progress_threads.c rename to opal/mca/pmix/pmix2x/pmix/src/util/progress_threads.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/progress_threads.h b/opal/mca/pmix/pmix2x/pmix/src/util/progress_threads.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/progress_threads.h rename to opal/mca/pmix/pmix2x/pmix/src/util/progress_threads.h diff --git a/opal/mca/pmix/pmix114/pmix/src/util/strnlen.h b/opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h similarity index 95% rename from opal/mca/pmix/pmix114/pmix/src/util/strnlen.h rename to opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h index f0dabe4950..ecb976207f 100644 --- a/opal/mca/pmix/pmix114/pmix/src/util/strnlen.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h @@ -26,7 +26,7 @@ int _x; \ (c) = 0; \ for (_x=0; _x < (b); _x++) { \ - if ('\0' == (a)[_x]) { \ + if ('\0' == (a)[_x]) { \ break; \ } \ ++(c); \ diff --git a/opal/mca/pmix/pmix114/pmix/src/util/timings.c b/opal/mca/pmix/pmix2x/pmix/src/util/timings.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/timings.c rename to opal/mca/pmix/pmix2x/pmix/src/util/timings.c diff --git a/opal/mca/pmix/pmix114/pmix/src/util/timings.h b/opal/mca/pmix/pmix2x/pmix/src/util/timings.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/src/util/timings.h rename to opal/mca/pmix/pmix2x/pmix/src/util/timings.h diff --git a/opal/mca/pmix/pmix114/pmix/test/Makefile.am b/opal/mca/pmix/pmix2x/pmix/test/Makefile.am similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/test/Makefile.am diff --git a/opal/mca/pmix/pmix114/pmix/test/README b/opal/mca/pmix/pmix2x/pmix/test/README similarity index 99% rename from opal/mca/pmix/pmix114/pmix/test/README rename to opal/mca/pmix/pmix2x/pmix/test/README index 15cd5f69ca..552558320c 100644 --- a/opal/mca/pmix/pmix114/pmix/test/README +++ b/opal/mca/pmix/pmix2x/pmix/test/README @@ -1,6 +1,6 @@ This is a short description how to run tests for pmix standalone library. -To compile test the user should run make in the test subdirectory. +To compile test the user should run make in the test subdirectory. To start testing, the user should run either pmix_test or pmix_test_lite executable. These applications are using two different versions of server functionality and fork diff --git a/opal/mca/pmix/pmix114/pmix/test/cli_stages.c b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c similarity index 91% rename from opal/mca/pmix/pmix114/pmix/test/cli_stages.c rename to opal/mca/pmix/pmix2x/pmix/test/cli_stages.c index b7a7444727..94a89304a0 100644 --- a/opal/mca/pmix/pmix114/pmix/test/cli_stages.c +++ b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -245,15 +245,19 @@ void cli_kill_all(void) } } -void errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) +void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) { - TEST_ERROR((" PMIX server Error handler with status = %d", status)); + TEST_ERROR((" PMIX server event handler with status = %d", status)); /* notify clients of error */ - PMIx_Notify_error(status, + PMIx_Notify_event(status, source, + PMIX_RANGE_NAMESPACE, NULL, 0, - NULL, 0, NULL, 0, op_callbk, NULL); } @@ -264,10 +268,10 @@ void op_callbk(pmix_status_t status, } void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, - void *cbdata) + size_t errhandler_ref, + void *cbdata) { - TEST_VERBOSE(("ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%d", - status, errhandler_ref)); + TEST_VERBOSE(("ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref)); } diff --git a/opal/mca/pmix/pmix114/pmix/test/cli_stages.h b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h similarity index 80% rename from opal/mca/pmix/pmix114/pmix/test/cli_stages.h rename to opal/mca/pmix/pmix2x/pmix/test/cli_stages.h index 441cb38ebf..37d418c032 100644 --- a/opal/mca/pmix/pmix114/pmix/test/cli_stages.h +++ b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -61,15 +61,19 @@ void cli_kill_all(void); bool test_terminated(void); -void errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo); +void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); void op_callbk(pmix_status_t status, void *cbdata); void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata); diff --git a/opal/mca/pmix/pmix114/pmix/test/pmi2_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c similarity index 90% rename from opal/mca/pmix/pmix114/pmix/test/pmi2_client.c rename to opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c index 22a444964a..92a27e99db 100644 --- a/opal/mca/pmix/pmix114/pmix/test/pmi2_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c @@ -62,8 +62,9 @@ static int test_item5(void); static int test_item6(void); static int test_item7(void); static int test_item8(void); -/* several sequence of fences is a buggy case for pmix v1.0 (see https://github.com/open-mpi/pmix/issues/37) */ static int test_item9(void); +/* several sequence of fences is a buggy case for pmix v1.0 (see https://github.com/open-mpi/pmix/issues/37) */ +static int test_item10(void); static int spawned, size, rank, appnum; static char jobid[100]; @@ -147,6 +148,12 @@ int main(int argc, char **argv) log_info("TI9 : %s\n", (rc ? "FAIL" : "PASS")); } + if (!ti || 10 == ti) { + rc = test_item10(); + ret += (rc ? 1 : 0); + log_info("TI10 : %s\n", (rc ? "FAIL" : "PASS")); + } + if (PMI2_SUCCESS != (rc = PMI2_Finalize())) { log_fatal("PMI2_Finalize failed: %d\n", rc); return rc; @@ -220,7 +227,7 @@ static int test_item3(void) const char **ptr = tkeys; if (_legacy || !_legacy) { - log_error("PMIx and SLURM/PMI2 does not set Job Attributes %s\n", "(Do not mark test as failed)"); + log_error("%s\n", "PMIx and SLURM/PMI2 does not set Job Attributes (Do not mark test as failed)"); return rc; } @@ -255,7 +262,7 @@ static int test_item4(void) const char **ptr = tkeys; if (_legacy || !_legacy) { - log_error("PMIx and SLURM/PMI2 does not set Node Attributes %s\n", "(Do not mark test as failed)"); + log_error("%s\n", "PMIx and SLURM/PMI2 does not set Node Attributes (Do not mark test as failed)"); return rc; } @@ -318,7 +325,7 @@ static int test_item6(void) rc = PMI2_KVS_Get(NULL, PMI2_ID_NULL, tkey, val, sizeof(val), &len); if (PMI2_SUCCESS == rc) { log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len); - log_error("PMI2_KVS_Get should not find data %s\n", "w/o commit"); + log_error("%s\n", "PMI2_KVS_Get should not find data w/o commit"); return 1; } @@ -326,6 +333,41 @@ static int test_item6(void) } static int test_item7(void) +{ + int rc = 0; + char val[PMI2_MAX_VALLEN]; + int len; + char tkey[PMI2_MAX_VALLEN]; + char tval[PMI2_MAX_VALLEN]; + + sprintf(tkey, "KEY-%d", rank); + sprintf(tval, "VALUE-%d", rank); + if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) { + log_fatal("PMI2_KVS_Put %d\n", rc); + return rc; + } + + if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) { + log_fatal("PMI2_KVS_Fence %d\n", rc); + return rc; + } + + /* expected result: return error status */ + rc = PMI2_KVS_Get(jobid, rank, tkey, val, sizeof(val), &len); + if (PMI2_SUCCESS != rc) { + log_fatal("PMI2_KVS_Get [%s=?] %d\n", tkey, rc); + return rc; + } + + log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len); + + log_assert((int)strlen(tval) == len, "value does not meet expectation"); + log_assert(!strcmp(tval, val), "value does not meet expectation"); + + return 0; +} + +static int test_item8(void) { int rc = 0; int len; @@ -367,7 +409,7 @@ static int test_item7(void) return rc; } -static int test_item8(void) +static int test_item9(void) { int rc = 0; int len; @@ -412,7 +454,7 @@ static int test_item8(void) return rc; } -static int test_item9(void) +static int test_item10(void) { int rc = 0; int i, j, r; diff --git a/opal/mca/pmix/pmix114/pmix/test/pmi_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmi_client.c similarity index 98% rename from opal/mca/pmix/pmix114/pmix/test/pmi_client.c rename to opal/mca/pmix/pmix2x/pmix/test/pmi_client.c index 9495a37976..70e72dfb1a 100644 --- a/opal/mca/pmix/pmix114/pmix/test/pmi_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmi_client.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2014 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -305,7 +305,7 @@ static int test_item5(void) const char **ptr = tkeys; if (_legacy || !_legacy) { - log_error("PMIx and SLURM/PMI1 do not set 'PMI_process_mapping' %s\n", "(Do not mark test as failed)"); + log_error("%s\n", "PMIx and SLURM/PMI1 do not set 'PMI_process_mapping' (Do not mark test as failed)"); return rc; } diff --git a/opal/mca/pmix/pmix114/pmix/test/pmix_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c similarity index 89% rename from opal/mca/pmix/pmix114/pmix/test/pmix_client.c rename to opal/mca/pmix/pmix2x/pmix/test/pmix_client.c index b9c648e1b0..5d05ca568f 100644 --- a/opal/mca/pmix/pmix114/pmix/test/pmix_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -41,9 +41,12 @@ #include "test_error.h" -static void errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) { TEST_ERROR(("PMIX client: Error handler with status = %d", status)) } @@ -55,11 +58,11 @@ static void op_callbk(pmix_status_t status, } static void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, - void *cbdata) + size_t errhandler_ref, + void *cbdata) { - TEST_VERBOSE(("PMIX client ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%d", - status, errhandler_ref)); + TEST_VERBOSE(("PMIX client ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref)); } int main(int argc, char **argv) @@ -82,12 +85,12 @@ int main(int argc, char **argv) } /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { TEST_ERROR(("Client ns %s rank %d: PMIx_Init failed: %d", params.nspace, params.rank, rc)); FREE_TEST_PARAMS(params); exit(0); } - PMIx_Register_errhandler(NULL, 0, errhandler, errhandler_reg_callbk, NULL); + PMIx_Register_event_handler(NULL, 0, NULL, 0, errhandler, errhandler_reg_callbk, NULL); if (myproc.rank != params.rank) { TEST_ERROR(("Client ns %s Rank returned in PMIx_Init %d does not match to rank from command line %d.", myproc.nspace, myproc.rank, params.rank)); FREE_TEST_PARAMS(params); @@ -189,18 +192,16 @@ int main(int argc, char **argv) } TEST_VERBOSE(("Client ns %s rank %d: PASSED", myproc.nspace, myproc.rank)); - PMIx_Deregister_errhandler(1, op_callbk, NULL); + PMIx_Deregister_event_handler(1, op_callbk, NULL); /* In case of direct modex we want to delay Finalize until everybody has finished. Otherwise some processes will fail to get data from others who already exited */ PMIx_Fence(NULL, 0, NULL, 0); - TEST_VERBOSE(("Client ns %s rank %d: PASSED", myproc.nspace, myproc.rank)); - PMIx_Deregister_errhandler(1, op_callbk, NULL); /* finalize us */ TEST_VERBOSE(("Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank)); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { TEST_ERROR(("Client ns %s rank %d:PMIx_Finalize failed: %d", myproc.nspace, myproc.rank, rc)); } else { TEST_VERBOSE(("Client ns %s rank %d:PMIx_Finalize successfully completed", myproc.nspace, myproc.rank)); diff --git a/opal/mca/pmix/pmix114/pmix/test/pmix_client_otheruser.sh b/opal/mca/pmix/pmix2x/pmix/test/pmix_client_otheruser.sh similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/pmix_client_otheruser.sh rename to opal/mca/pmix/pmix2x/pmix/test/pmix_client_otheruser.sh diff --git a/opal/mca/pmix/pmix114/pmix/test/pmix_regex.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c similarity index 99% rename from opal/mca/pmix/pmix114/pmix/test/pmix_regex.c rename to opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c index ece191ebc6..027f6fc5a3 100644 --- a/opal/mca/pmix/pmix114/pmix/test/pmix_regex.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c @@ -42,7 +42,7 @@ int main(int argc, char **argv) char *regex; char **nodes, **procs; pmix_status_t rc; - + /* smoke test */ if (PMIX_SUCCESS != 0) { TEST_ERROR(("ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d", PMIX_SUCCESS)); @@ -68,7 +68,7 @@ int main(int argc, char **argv) } else { fprintf(stderr, "Node reverse failed: %d\n\n\n", rc); } - + fprintf(stderr, "PROCS: %s\n", TEST_PROCS); PMIx_generate_ppn(TEST_PROCS, ®ex); fprintf(stderr, "PPN: %s\n\n", regex); diff --git a/opal/mca/pmix/pmix114/pmix/test/pmix_test.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_test.c similarity index 96% rename from opal/mca/pmix/pmix114/pmix/test/pmix_test.c rename to opal/mca/pmix/pmix2x/pmix/test/pmix_test.c index 1e6c468cea..9594b6c819 100644 --- a/opal/mca/pmix/pmix114/pmix/test/pmix_test.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_test.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. @@ -92,7 +92,8 @@ int main(int argc, char **argv) return rc; } /* register the errhandler */ - PMIx_Register_errhandler(NULL, 0, errhandler, errhandler_reg_callbk, NULL); + PMIx_Register_event_handler(NULL, 0, NULL, 0, + errhandler, errhandler_reg_callbk, NULL); order[CLI_UNINIT] = CLI_FORKED; order[CLI_FORKED] = CLI_FIN; @@ -178,7 +179,7 @@ int main(int argc, char **argv) pmix_argv_free(client_env); /* deregister the errhandler */ - PMIx_Deregister_errhandler(0, op_callbk, NULL); + PMIx_Deregister_event_handler(0, op_callbk, NULL); cli_wait_all(1.0); diff --git a/opal/mca/pmix/pmix114/pmix/test/server_callbacks.c b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c similarity index 90% rename from opal/mca/pmix/pmix114/pmix/test/server_callbacks.c rename to opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c index fd89729216..7b925db9ba 100644 --- a/opal/mca/pmix/pmix114/pmix/test/server_callbacks.c +++ b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. @@ -17,20 +17,19 @@ #include "src/util/argv.h" pmix_server_module_t mymodule = { - connected, - finalized, - abort_fn, - fencenb_fn, - dmodex_fn, - publish_fn, - lookup_fn, - unpublish_fn, - spawn_fn, - connect_fn, - disconnect_fn, - regevents_fn, - deregevents_fn, - NULL + .client_connected = connected, + .client_finalized = finalized, + .abort = abort_fn, + .fence_nb = fencenb_fn, + .direct_modex = dmodex_fn, + .publish = publish_fn, + .lookup = lookup_fn, + .unpublish = unpublish_fn, + .spawn = spawn_fn, + .connect = connect_fn, + .disconnect = disconnect_fn, + .register_events = regevents_fn, + .deregister_events = deregevents_fn }; typedef struct { @@ -80,8 +79,12 @@ pmix_list_t *pmix_test_published_list = NULL; static int finalized_count = 0; -pmix_status_t connected(const pmix_proc_t *proc, void *server_object) +pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) { + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } return PMIX_SUCCESS; } @@ -303,8 +306,9 @@ pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, return PMIX_SUCCESS; } -pmix_status_t regevents_fn (const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) +pmix_status_t regevents_fn (pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) { TEST_VERBOSE ((" pmix host server regevents_fn called ")); if (NULL != cbfunc) { @@ -313,8 +317,8 @@ pmix_status_t regevents_fn (const pmix_info_t info[], size_t ninfo, return PMIX_SUCCESS; } -pmix_status_t deregevents_fn (const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) +pmix_status_t deregevents_fn (pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata) { TEST_VERBOSE ((" pmix host server deregevents_fn called ")); if (NULL != cbfunc) { diff --git a/opal/mca/pmix/pmix114/pmix/test/server_callbacks.h b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h similarity index 86% rename from opal/mca/pmix/pmix114/pmix/test/server_callbacks.h rename to opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h index bea2e8ab79..df0d5ca577 100644 --- a/opal/mca/pmix/pmix114/pmix/test/server_callbacks.h +++ b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -15,7 +15,8 @@ #include "cli_stages.h" -pmix_status_t connected(const pmix_proc_t *proc, void *server_object); +pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); pmix_status_t abort_fn(const pmix_proc_t *proc, @@ -49,10 +50,11 @@ pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); -pmix_status_t regevents_fn(const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); -pmix_status_t deregevents_fn(const pmix_info_t info[], size_t ninfo, +pmix_status_t regevents_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); +pmix_status_t deregevents_fn(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); extern pmix_server_module_t mymodule; #endif diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/Makefile.am b/opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am similarity index 92% rename from opal/mca/pmix/pmix114/pmix/test/simple/Makefile.am rename to opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am index 2d7506cb7f..c5b8731994 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am @@ -21,7 +21,7 @@ AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix -noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex +noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex test_pmix simptest_SOURCES = \ simptest.c @@ -58,3 +58,9 @@ simpdyn_SOURCES = \ simpdyn_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpdyn_LDADD = \ $(top_builddir)/libpmix.la + +test_pmix_SOURCES = \ + test_pmix.c +test_pmix_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +test_pmix_LDADD = \ + $(top_builddir)/libpmix.la diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/simpclient.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c similarity index 78% rename from opal/mca/pmix/pmix114/pmix/test/simple/simpclient.c rename to opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c index 1554428f84..c71e3469c7 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/simpclient.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c @@ -38,23 +38,68 @@ #define MAXCNT 2 +static volatile bool completed = false; +static pmix_proc_t myproc; + +static void notification_fn(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + pmix_output(0, "Client %s:%d NOTIFIED with status %s", myproc.nspace, myproc.rank, PMIx_Error_string(status)); + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + completed = true; +} + +static void errhandler_reg_callbk(pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + volatile bool *active = (volatile bool*)cbdata; + + pmix_output(0, "Client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref); + *active = false; +} + int main(int argc, char **argv) { int rc; pmix_value_t value; pmix_value_t *val = &value; char *tmp; - pmix_proc_t proc, myproc; + pmix_proc_t proc; uint32_t nprocs, n; int cnt, j; + bool doabort = false; + volatile bool active; + + if (1 < argc) { + if (0 == strcmp("-abort", argv[1])) { + doabort = true; + } + } /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); exit(rc); } pmix_output(0, "Client ns %s rank %d: Running", myproc.nspace, myproc.rank); + /* register our errhandler */ + active = true; + PMIx_Register_event_handler(NULL, 0, NULL, 0, + notification_fn, errhandler_reg_callbk, (void*)&active); + while (active) { + usleep(10); + } + /* get our universe size */ if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_UNIV_SIZE, NULL, 0, &val))) { pmix_output(0, "Client ns %s rank %d: PMIx_Get universe size failed: %d", myproc.nspace, myproc.rank, rc); @@ -154,10 +199,21 @@ int main(int argc, char **argv) } } + /* if requested and our rank is 0, call abort */ + if (doabort) { + if (0 == myproc.rank) { + PMIx_Abort(PMIX_ERR_PROC_REQUESTED_ABORT, "CALLING ABORT", NULL, 0); + } else { + while(!completed) { + usleep(10); + } + } + } + done: /* finalize us */ pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/simpdmodex.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/test/simple/simpdmodex.c rename to opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c index ca60541b84..05dd8acebf 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/simpdmodex.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -97,7 +97,7 @@ int main(int argc, char **argv) bool active; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); exit(0); } @@ -200,7 +200,7 @@ int main(int argc, char **argv) done: /* finalize us */ pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/simpdyn.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/test/simple/simpdyn.c rename to opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c index a0a60e99d0..99987e606a 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/simpdyn.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -57,7 +57,7 @@ int main(int argc, char **argv) gethostname(hostname, sizeof(hostname)); /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); exit(0); } @@ -71,7 +71,7 @@ int main(int argc, char **argv) nprocs = val->data.uint32; PMIX_VALUE_RELEASE(val); pmix_output(0, "Client %s:%d universe size %d", myproc.nspace, myproc.rank, nprocs); - + /* call fence to sync */ PMIX_PROC_CONSTRUCT(&proc); (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); @@ -80,7 +80,7 @@ int main(int argc, char **argv) pmix_output(0, "Client ns %s rank %d: PMIx_Fence failed: %d", myproc.nspace, myproc.rank, rc); goto done; } - + /* rank=0 calls spawn */ if (0 == myproc.rank) { PMIX_APP_CREATE(app, 1); @@ -172,7 +172,7 @@ int main(int argc, char **argv) } PMIX_PROC_FREE(peers, npeers); free(nodelist); - + done: /* call fence to sync */ PMIX_PROC_CONSTRUCT(&proc); @@ -182,11 +182,11 @@ int main(int argc, char **argv) pmix_output(0, "Client ns %s rank %d: PMIx_Fence failed: %d", myproc.nspace, myproc.rank, rc); goto done; } - + /* finalize us */ pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); - - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/simpft.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpft.c similarity index 79% rename from opal/mca/pmix/pmix114/pmix/test/simple/simpft.c rename to opal/mca/pmix/pmix2x/pmix/test/simple/simpft.c index cf71588630..afe2a97523 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/simpft.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpft.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -40,11 +40,18 @@ static pmix_proc_t myproc; static bool completed; -static void notification_fn(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) +static void notification_fn(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) { pmix_output(0, "Client %s:%d NOTIFIED with status %d", myproc.nspace, myproc.rank, status); + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } completed = true; } @@ -55,11 +62,11 @@ static void op_callbk(pmix_status_t status, } static void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata) { - pmix_output(0, "Client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%d", - status, errhandler_ref); + pmix_output(0, "Client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref); } int main(int argc, char **argv) @@ -71,7 +78,7 @@ int main(int argc, char **argv) uint32_t nprocs; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); exit(0); } @@ -88,7 +95,8 @@ int main(int argc, char **argv) completed = false; /* register our errhandler */ - PMIx_Register_errhandler(NULL, 0, notification_fn, errhandler_reg_callbk, NULL); + PMIx_Register_event_handler(NULL, 0, NULL, 0, + notification_fn, errhandler_reg_callbk, NULL); /* call fence to sync */ PMIX_PROC_CONSTRUCT(&proc); @@ -116,9 +124,9 @@ int main(int argc, char **argv) done: /* finalize us */ pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); - PMIx_Deregister_errhandler(0, op_callbk, NULL); + PMIx_Deregister_event_handler(1, op_callbk, NULL); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/simppub.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simppub.c similarity index 97% rename from opal/mca/pmix/pmix114/pmix/test/simple/simppub.c rename to opal/mca/pmix/pmix2x/pmix/test/simple/simppub.c index 1b866710a0..42b07df810 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/simppub.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simppub.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -49,7 +49,7 @@ int main(int argc, char **argv) pmix_proc_t myproc; /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc))) { + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); exit(0); } @@ -157,7 +157,7 @@ int main(int argc, char **argv) done: /* finalize us */ pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Finalize())) { + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); } else { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); diff --git a/opal/mca/pmix/pmix114/pmix/test/simple/simptest.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c similarity index 80% rename from opal/mca/pmix/pmix114/pmix/test/simple/simptest.c rename to opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c index f6ff0b6467..a950dbb41f 100644 --- a/opal/mca/pmix/pmix114/pmix/test/simple/simptest.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -46,7 +46,8 @@ #include "src/buffer_ops/buffer_ops.h" #include "src/usock/usock.h" -static pmix_status_t connected(const pmix_proc_t *proc, void *server_object); +static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t abort_fn(const pmix_proc_t *proc, void *server_object, @@ -79,23 +80,32 @@ static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t register_event_fn(const pmix_info_t info[], size_t ninfo, +static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_server_module_t mymodule = { - connected, - finalized, - abort_fn, - fencenb_fn, - dmodex_fn, - publish_fn, - lookup_fn, - unpublish_fn, - spawn_fn, - connect_fn, - disconnect_fn, - register_event_fn, - NULL + .client_connected = connected, + .client_finalized = finalized, + .abort = abort_fn, + .fence_nb = fencenb_fn, + .direct_modex = dmodex_fn, + .publish = publish_fn, + .lookup = lookup_fn, + .unpublish = unpublish_fn, + .spawn = spawn_fn, + .connect = connect_fn, + .disconnect = disconnect_fn, + .register_events = register_event_fn, + .deregister_events = deregister_events, + .notify_event = notify_event }; typedef struct { @@ -150,13 +160,16 @@ static pmix_list_t children; static void set_namespace(int nprocs, char *ranks, char *nspace, pmix_op_cbfunc_t cbfunc, myxfer_t *x); -static void errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo); +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); static void wait_signal_callback(int fd, short event, void *arg); -static void op_callbk(pmix_status_t status, void *cbdata); static void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata); static void opcbfunc(pmix_status_t status, void *cbdata) @@ -175,7 +188,7 @@ int main(int argc, char **argv) char **client_env=NULL; char **client_argv=NULL; char *tmp, **atmp, *executable=NULL; - int rc, nprocs=1, n; + int rc, nprocs=1, n, k; uid_t myuid; gid_t mygid; pid_t pid; @@ -197,7 +210,8 @@ int main(int argc, char **argv) return rc; } /* register the errhandler */ - PMIx_Register_errhandler(NULL, 0, errhandler, errhandler_reg_callbk, NULL); + PMIx_Register_event_handler(NULL, 0, NULL, 0, + errhandler, errhandler_reg_callbk, NULL); /* setup the pub data, in case it is used */ PMIX_CONSTRUCT(&pubdata, pmix_list_t); @@ -218,7 +232,10 @@ int main(int argc, char **argv) } else if (0 == strcmp("-e", argv[n]) && NULL != argv[n+1]) { executable = strdup(argv[n+1]); - ++n; + for (k=n+2; NULL != argv[k]; k++) { + pmix_argv_append_nosize(&client_argv, argv[k]); + } + n += k; } } if (NULL == executable) { @@ -238,7 +255,7 @@ int main(int argc, char **argv) /* set common argv and env */ client_env = pmix_argv_copy(environ); - pmix_argv_append_nosize(&client_argv, executable); + pmix_argv_prepend_nosize(&client_argv, executable); wakeup = nprocs; myuid = getuid(); @@ -298,7 +315,7 @@ int main(int argc, char **argv) pmix_argv_free(client_env); /* deregister the errhandler */ - PMIx_Deregister_errhandler(0, op_callbk, NULL); + PMIx_Deregister_event_handler(0, NULL, NULL); /* release any pub data */ PMIX_LIST_DESTRUCT(&pubdata); @@ -353,29 +370,31 @@ static void set_namespace(int nprocs, char *ranks, char *nspace, cbfunc, x); } -static void errhandler(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) { pmix_output(0, "SERVER: ERRHANDLER CALLED WITH STATUS %d", status); } -static void op_callbk(pmix_status_t status, - void *cbdata) -{ - pmix_output(0, "SERVER: OP CALLBACK CALLED WITH STATUS %d", status); -} - static void errhandler_reg_callbk (pmix_status_t status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata) { - pmix_output(0, "SERVER: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%d", - status, errhandler_ref); + pmix_output(0, "SERVER: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref); } -static pmix_status_t connected(const pmix_proc_t *proc, void *server_object) +static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) { + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } return PMIX_SUCCESS; } static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, @@ -403,15 +422,19 @@ static void abcbfunc(pmix_status_t status, void *cbdata) } static pmix_status_t abort_fn(const pmix_proc_t *proc, - void *server_object, - int status, const char msg[], - pmix_proc_t procs[], size_t nprocs, - pmix_op_cbfunc_t cbfunc, void *cbdata) + void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t rc; myxfer_t *x; - pmix_output(0, "SERVER: ABORT on %s:%d", procs[0].nspace, procs[0].rank); + if (NULL != procs) { + pmix_output(0, "SERVER: ABORT on %s:%d", procs[0].nspace, procs[0].rank); + } else { + pmix_output(0, "SERVER: ABORT OF ALL PROCS IN NSPACE %s", proc->nspace); + } /* instead of aborting the specified procs, notify them * (if they have registered their errhandler) */ @@ -432,9 +455,10 @@ static pmix_status_t abort_fn(const pmix_proc_t *proc, x->cbfunc = cbfunc; x->cbdata = cbdata; - if (PMIX_SUCCESS != (rc = PMIx_Notify_error(status, procs, nprocs, - &x->caller, 1, x->info, 2, - abcbfunc, x))) { + if (PMIX_SUCCESS != (rc = PMIx_Notify_event(status, &x->caller, + PMIX_RANGE_NAMESPACE, + x->info, 2, + abcbfunc, x))) { pmix_output(0, "SERVER: FAILED NOTIFY ERROR %d", (int)rc); } @@ -527,10 +551,12 @@ static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, PMIX_PDATA_CREATE(pd, n); for (i=0; i < n; i++) { p = (pmix_locdat_t*)pmix_list_remove_first(&results); - (void)strncpy(pd[i].proc.nspace, p->pdata.proc.nspace, PMIX_MAX_NSLEN); - pd[i].proc.rank = p->pdata.proc.rank; - (void)strncpy(pd[i].key, p->pdata.key, PMIX_MAX_KEYLEN); - pmix_value_xfer(&pd[i].value, &p->pdata.value); + if (p) { + (void)strncpy(pd[i].proc.nspace, p->pdata.proc.nspace, PMIX_MAX_NSLEN); + pd[i].proc.rank = p->pdata.proc.rank; + (void)strncpy(pd[i].key, p->pdata.key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&pd[i].value, &p->pdata.value); + } } } PMIX_LIST_DESTRUCT(&results); @@ -604,8 +630,8 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc, static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_output(0, "SERVER: CONNECT"); @@ -621,8 +647,8 @@ static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_output(0, "SERVER: DISCONNECT"); @@ -636,12 +662,28 @@ static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, return PMIX_SUCCESS; } -static pmix_status_t register_event_fn(const pmix_info_t info[], size_t ninfo, +static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { return PMIX_SUCCESS; } +static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return PMIX_SUCCESS; +} + static void wait_signal_callback(int fd, short event, void *arg) { pmix_event_t *sig = (pmix_event_t*) arg; diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c b/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c new file mode 100644 index 0000000000..8ecf6c1f6b --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c @@ -0,0 +1,67 @@ + +#include +#include +#include + +int main(int argc, char **argv) +{ + pmix_proc_t myproc; + pmix_status_t rc; + + int rank; + rc = PMIx_Init(&myproc, NULL, 0); + assert(PMIX_SUCCESS == rc); + + { + pmix_value_t *value; + rc = PMIx_Get(&myproc, PMIX_RANK, NULL, 0, &value); + assert(PMIX_SUCCESS == rc); + printf("%d\n", value->type); + assert(value->type == PMIX_INT); + rank = value->data.uint32; + PMIX_VALUE_RELEASE(value); + } + + if (rank == 0 ) { + pmix_info_t *info; + PMIX_INFO_CREATE(info, 1); + snprintf(info[0].key, PMIX_MAX_KEYLEN, "magic-found"); + info[0].value.type = PMIX_STRING; + info[0].value.data.string = "yes"; + rc = PMIx_Publish(info, 1); + assert(PMIX_SUCCESS == rc); + } + + printf("I am rank %d\n", rank); + + { + bool flag; + pmix_info_t *info; + PMIX_INFO_CREATE(info, 1); + flag = true; + PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL); + rc = PMIx_Fence(&myproc, 1, info, 1); + assert(PMIX_SUCCESS == rc); + PMIX_INFO_FREE(info, 1); + } + + if (rank == 1) { + int i; + pmix_pdata_t *pdata; + PMIX_PDATA_CREATE(pdata, 2); + snprintf(pdata[0].key, PMIX_MAX_KEYLEN, "magic-found"); + snprintf(pdata[1].key, PMIX_MAX_KEYLEN, "magic-not-found"); + rc = PMIx_Lookup(&pdata[0], 2, NULL, 0); + assert((PMIX_SUCCESS == rc) || (PMIX_ERR_NOT_FOUND == rc)); + for ( i = 0 ; i < 2 ; i++ ) + if (pdata[i].value.type == PMIX_STRING) + printf("Found[%d] %d %s\n", i, pdata[i].value.type, pdata[i].value.data.string); + else + printf("Found[%d] %d\n", i, pdata[i].value.type); + PMIX_PDATA_FREE(pdata, 1); + } + + rc = PMIx_Finalize(NULL, 0); + assert(PMIX_SUCCESS == rc); + +} diff --git a/opal/mca/pmix/pmix114/pmix/test/test_cd.c b/opal/mca/pmix/pmix2x/pmix/test/test_cd.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_cd.c rename to opal/mca/pmix/pmix2x/pmix/test/test_cd.c diff --git a/opal/mca/pmix/pmix114/pmix/test/test_cd.h b/opal/mca/pmix/pmix2x/pmix/test/test_cd.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_cd.h rename to opal/mca/pmix/pmix2x/pmix/test/test_cd.h diff --git a/opal/mca/pmix/pmix114/pmix/test/test_common.c b/opal/mca/pmix/pmix2x/pmix/test/test_common.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_common.c rename to opal/mca/pmix/pmix2x/pmix/test/test_common.c diff --git a/opal/mca/pmix/pmix114/pmix/test/test_common.h b/opal/mca/pmix/pmix2x/pmix/test/test_common.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_common.h rename to opal/mca/pmix/pmix2x/pmix/test/test_common.h diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_error.c b/opal/mca/pmix/pmix2x/pmix/test/test_error.c new file mode 100644 index 0000000000..f5217f0657 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/test_error.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#include +#include "test_error.h" +#include "test_common.h" + +#define MAX_ERR_HANDLERS 5 +#define TEST_NOTIFY PMIX_ERR_TIMEOUT +static bool done; +static void comfail_errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + TEST_ERROR(("comfail errhandler called for error status = %d ninfo = %lu", + status, (unsigned long)ninfo)); + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } +} + +static void timeout_errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + TEST_ERROR(("timeout errhandler called for error status = %d ninfo = %d", + status, ninfo)); + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } +} + +static void op1_callbk(pmix_status_t status, + void *cbdata) +{ + TEST_VERBOSE(( "op1_callbk CALLED WITH STATUS %d", status)); + done = true; +} + +static void errhandler_reg_callbk1 (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + size_t *ref = (size_t*) cbdata; + *ref = errhandler_ref; + TEST_VERBOSE(("PMIX client ERRHANDLER REGISTRATION CALLED WITH STATUS %d, ref=%lu", + status, *ref, (unsigned long)errhandler_ref)); + +} + +int test_error(char *my_nspace, int my_rank, test_params params) +{ + size_t errhandler_refs[MAX_ERR_HANDLERS]; + struct timespec ts; + pmix_status_t status; + pmix_proc_t source; + + TEST_VERBOSE(("test-error: running error handling test cases")); + /* register specific client error handlers and test their invocation + * by trigerring events from server side*/ + status = PMIX_ERR_TIMEOUT; + PMIx_Register_event_handler(&status, 1, NULL, 0, + timeout_errhandler, errhandler_reg_callbk1, &errhandler_refs[0]); + + /* reg a handler for comm errors */ + status = PMIX_ERR_LOST_PEER_CONNECTION; + PMIx_Register_event_handler(&status, 1, NULL, 0, + comfail_errhandler, errhandler_reg_callbk1, &errhandler_refs[1]); + /* inject error from client */ + done = false; + (void)strncpy(source.nspace, my_nspace, PMIX_MAX_NSLEN); + source.rank = my_rank; + /* change error value to test other error notifications */ + PMIx_Notify_event(TEST_NOTIFY, + &source, PMIX_RANGE_NAMESPACE, + NULL, 0, + op1_callbk, NULL); + while(!done) { + ts.tv_sec = 0; + ts.tv_nsec = 100000; + nanosleep(&ts, NULL); + } + done = false; + /* dereg all handlers*/ + PMIx_Deregister_event_handler( errhandler_refs[0], op1_callbk, NULL); + /* loop until we get callback */ + while(!done) { + ts.tv_sec = 0; + ts.tv_nsec = 100000; + nanosleep(&ts, NULL); + } + done = false; + PMIx_Deregister_event_handler( errhandler_refs[1], op1_callbk, NULL); + /* loop until we get callback */ + while(!done) { + ts.tv_sec = 0; + ts.tv_nsec = 100000; + nanosleep(&ts, NULL); + } + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix114/pmix/test/test_error.h b/opal/mca/pmix/pmix2x/pmix/test/test_error.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_error.h rename to opal/mca/pmix/pmix2x/pmix/test/test_error.h diff --git a/opal/mca/pmix/pmix114/pmix/test/test_fence.c b/opal/mca/pmix/pmix2x/pmix/test/test_fence.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_fence.c rename to opal/mca/pmix/pmix2x/pmix/test/test_fence.c diff --git a/opal/mca/pmix/pmix114/pmix/test/test_fence.h b/opal/mca/pmix/pmix2x/pmix/test/test_fence.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_fence.h rename to opal/mca/pmix/pmix2x/pmix/test/test_fence.h diff --git a/opal/mca/pmix/pmix114/pmix/test/test_publish.c b/opal/mca/pmix/pmix2x/pmix/test/test_publish.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_publish.c rename to opal/mca/pmix/pmix2x/pmix/test/test_publish.c diff --git a/opal/mca/pmix/pmix114/pmix/test/test_publish.h b/opal/mca/pmix/pmix2x/pmix/test/test_publish.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_publish.h rename to opal/mca/pmix/pmix2x/pmix/test/test_publish.h diff --git a/opal/mca/pmix/pmix114/pmix/test/test_resolve_peers.c b/opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_resolve_peers.c rename to opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.c diff --git a/opal/mca/pmix/pmix114/pmix/test/test_resolve_peers.h b/opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_resolve_peers.h rename to opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.h diff --git a/opal/mca/pmix/pmix114/pmix/test/test_spawn.c b/opal/mca/pmix/pmix2x/pmix/test/test_spawn.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_spawn.c rename to opal/mca/pmix/pmix2x/pmix/test/test_spawn.c diff --git a/opal/mca/pmix/pmix114/pmix/test/test_spawn.h b/opal/mca/pmix/pmix2x/pmix/test/test_spawn.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/test_spawn.h rename to opal/mca/pmix/pmix2x/pmix/test/test_spawn.h diff --git a/opal/mca/pmix/pmix114/pmix/test/utils.c b/opal/mca/pmix/pmix2x/pmix/test/utils.c similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/utils.c rename to opal/mca/pmix/pmix2x/pmix/test/utils.c diff --git a/opal/mca/pmix/pmix114/pmix/test/utils.h b/opal/mca/pmix/pmix2x/pmix/test/utils.h similarity index 100% rename from opal/mca/pmix/pmix114/pmix/test/utils.h rename to opal/mca/pmix/pmix2x/pmix/test/utils.h diff --git a/opal/mca/pmix/pmix2x/pmix2x.c b/opal/mca/pmix/pmix2x/pmix2x.c new file mode 100644 index 0000000000..70904a3f72 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix2x.c @@ -0,0 +1,1172 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" +#include "opal/types.h" + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "opal/dss/dss.h" +#include "opal/mca/event/event.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_progress_threads.h" +#include "opal/util/argv.h" +#include "opal/util/error.h" +#include "opal/util/output.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" + +#include "pmix2x.h" +#include "opal/mca/pmix/base/base.h" +#include "opal/mca/pmix/pmix_types.h" + +#include + +/**** C.O.M.M.O.N I.N.T.E.R.F.A.C.E.S ****/ + +/* These are functions used by both client and server to + * access common functions in the embedded PMIx library */ + +static const char *pmix2x_get_nspace(opal_jobid_t jobid); +static void pmix2x_register_jobid(opal_jobid_t jobid, const char *nspace); +static void register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, + void *cbdata); +static void deregister_handler(size_t evhandler, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +static int notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); + +const opal_pmix_base_module_t opal_pmix_pmix2x_module = { + /* client APIs */ + .init = pmix2x_client_init, + .finalize = pmix2x_client_finalize, + .initialized = pmix2x_initialized, + .abort = pmix2x_abort, + .commit = pmix2x_commit, + .fence = pmix2x_fence, + .fence_nb = pmix2x_fencenb, + .put = pmix2x_put, + .get = pmix2x_get, + .get_nb = pmix2x_getnb, + .publish = pmix2x_publish, + .publish_nb = pmix2x_publishnb, + .lookup = pmix2x_lookup, + .lookup_nb = pmix2x_lookupnb, + .unpublish = pmix2x_unpublish, + .unpublish_nb = pmix2x_unpublishnb, + .spawn = pmix2x_spawn, + .spawn_nb = pmix2x_spawnnb, + .connect = pmix2x_connect, + .connect_nb = pmix2x_connectnb, + .disconnect = pmix2x_disconnect, + .disconnect_nb = pmix2x_disconnectnb, + .resolve_peers = pmix2x_resolve_peers, + .resolve_nodes = pmix2x_resolve_nodes, + /* server APIs */ + .server_init = pmix2x_server_init, + .server_finalize = pmix2x_server_finalize, + .generate_regex = pmix2x_server_gen_regex, + .generate_ppn = pmix2x_server_gen_ppn, + .server_register_nspace = pmix2x_server_register_nspace, + .server_deregister_nspace = pmix2x_server_deregister_nspace, + .server_register_client = pmix2x_server_register_client, + .server_deregister_client = pmix2x_server_deregister_client, + .server_setup_fork = pmix2x_server_setup_fork, + .server_dmodex_request = pmix2x_server_dmodex, + .server_notify_event = pmix2x_server_notify_event, + /* utility APIs */ + .get_version = PMIx_Get_version, + .register_evhandler = register_handler, + .deregister_evhandler = deregister_handler, + .notify_event = notify_event, + .store_local = pmix2x_store_local, + .get_nspace = pmix2x_get_nspace, + .register_jobid = pmix2x_register_jobid +}; + +static const char *pmix2x_get_nspace(opal_jobid_t jobid) +{ + opal_pmix2x_jobid_trkr_t *jptr; + + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { + if (jptr->jobid == jobid) { + return jptr->nspace; + } + } + return NULL; +} + +static void pmix2x_register_jobid(opal_jobid_t jobid, const char *nspace) +{ + opal_pmix2x_jobid_trkr_t *jptr; + + /* if we don't already have it, add this to our jobid tracker */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { + if (jptr->jobid == jobid) { + return; + } + } + jptr = OBJ_NEW(opal_pmix2x_jobid_trkr_t); + (void)strncpy(jptr->nspace, nspace, PMIX_MAX_NSLEN); + jptr->jobid = jobid; + opal_list_append(&mca_pmix_pmix2x_component.jobids, &jptr->super); +} + +static void completion_handler(int status, void *cbdata) +{ + opal_pmix2x_event_chain_t *chain = (opal_pmix2x_event_chain_t*)cbdata; + if (NULL != chain->info) { + OPAL_LIST_RELEASE(chain->info); + } +} + +static void progress_local_event_hdlr(int status, + opal_list_t *results, + opal_pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata) +{ + opal_pmix2x_event_chain_t *chain = (opal_pmix2x_event_chain_t*)notification_cbdata; + size_t n; + opal_list_item_t *nxt; + opal_pmix2x_single_event_t *sing; + opal_pmix2x_multi_event_t *multi; + opal_pmix2x_default_event_t *def; + + /* if the caller indicates that the chain is completed, then stop here */ + if (OPAL_ERR_HANDLERS_COMPLETE == status) { + goto complete; + } + + /* if any results were provided, then add them here */ + if (NULL != results) { + while (NULL != (nxt = opal_list_remove_first(results))) { + opal_list_append(results, nxt); + } + } + + /* see if we need to continue, starting with the single code events */ + if (NULL != chain->sing) { + /* the last handler was for a single code - see if there are + * any others that match this event */ + while (opal_list_get_end(&mca_pmix_pmix2x_component.single_events) != (nxt = opal_list_get_next(&chain->sing->super))) { + sing = (opal_pmix2x_single_event_t*)nxt; + if (sing->code == chain->status) { + OBJ_RETAIN(chain); + chain->sing = sing; + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + goto complete; + } + } + /* if we get here, then there are no more single code + * events that match */ + chain->sing = NULL; + /* pickup the beginning of the multi-code event list */ + if (0 < opal_list_get_size(&mca_pmix_pmix2x_component.multi_events)) { + chain->multi = (opal_pmix2x_multi_event_t*)opal_list_get_begin(&mca_pmix_pmix2x_component.multi_events); + } + } + + /* see if we need to continue with the multi code events */ + if (NULL != chain->multi) { + while (opal_list_get_end(&mca_pmix_pmix2x_component.multi_events) != (nxt = opal_list_get_next(&chain->multi->super))) { + multi = (opal_pmix2x_multi_event_t*)nxt; + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + OBJ_RETAIN(chain); + chain->multi = multi; + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + goto complete; + } + } + } + /* if we get here, then there are no more multi-mode + * events that match */ + chain->multi = NULL; + /* pickup the beginning of the default event list */ + if (0 < opal_list_get_size(&mca_pmix_pmix2x_component.default_events)) { + chain->def = (opal_pmix2x_default_event_t*)opal_list_get_begin(&mca_pmix_pmix2x_component.default_events); + } + } + + /* if they didn't want it to go to a default handler, then we are done */ + if (chain->nondefault) { + goto complete; + } + + if (NULL != chain->def) { + if (opal_list_get_end(&mca_pmix_pmix2x_component.default_events) != (nxt = opal_list_get_next(&chain->def->super))) { + def = (opal_pmix2x_default_event_t*)nxt; + OBJ_RETAIN(chain); + chain->def = def; + def->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + } + } + + complete: + /* we still have to call their final callback */ + if (NULL != chain->final_cbfunc) { + chain->final_cbfunc(OPAL_SUCCESS, chain->final_cbdata); + } + /* maintain acctng */ + OBJ_RELEASE(chain); + /* let the caller know that we are done with their callback */ + if (NULL != cbfunc) { + cbfunc(OPAL_SUCCESS, thiscbdata); + } +} + +static void _event_hdlr(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + size_t n; + opal_pmix2x_event_chain_t *chain; + opal_pmix2x_single_event_t *sing; + opal_pmix2x_multi_event_t *multi; + opal_pmix2x_default_event_t *def; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s RECEIVED NOTIFICATION OF STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), cd->status); + + chain = OBJ_NEW(opal_pmix2x_event_chain_t); + /* point it at our final callback */ + chain->final_cbfunc = completion_handler; + chain->final_cbdata = chain; + + /* carry across provided info */ + chain->status = cd->status; + chain->source = cd->pname; + chain->info = cd->info; + chain->nondefault = cd->nondefault; + + /* cycle thru the single-event registrations first */ + OPAL_LIST_FOREACH(sing, &mca_pmix_pmix2x_component.single_events, opal_pmix2x_single_event_t) { + if (sing->code == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + OBJ_RETAIN(chain); + chain->sing = sing; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CALLING SINGLE EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + return; + } + } + + /* if we didn't find any match in the single-event registrations, + * then cycle thru the multi-event registrations next */ + OPAL_LIST_FOREACH(multi, &mca_pmix_pmix2x_component.multi_events, opal_pmix2x_multi_event_t) { + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + OBJ_RETAIN(chain); + chain->multi = multi; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CALLING MULTI EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + return; + } + } + } + + /* if they didn't want it to go to a default handler, then we are done */ + if (chain->nondefault) { + /* if we get here, then we need to cache this event in case they + * register for it later - we cannot lose individual events */ + opal_list_append(&mca_pmix_pmix2x_component.cache, &chain->super); + return; + } + + /* we are done with the threadshift caddy */ + OBJ_RELEASE(cd); + + /* finally, pass it to any default handlers */ + if (0 < opal_list_get_size(&mca_pmix_pmix2x_component.default_events)) { + def = (opal_pmix2x_default_event_t*)opal_list_get_first(&mca_pmix_pmix2x_component.default_events); + OBJ_RETAIN(chain); + chain->def = def; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CALLING DEFAULT EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + def->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + return; + } + + /* we still have to call their final callback */ + if (NULL != chain->final_cbfunc) { + chain->final_cbfunc(PMIX_SUCCESS, chain->final_cbdata); + } + return; +} + +/* this function will be called by the PMIx client library + * whenever it receives notification of an event. The + * notification can come from an ORTE daemon (when launched + * by mpirun), directly from a RM (when direct launched), or + * from another process (via the local daemon). + * The call will occur in the PMIx event base */ +void pmix2x_event_hdlr(size_t evhdlr_registration_id, + pmix_status_t status, const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + pmix2x_threadshift_t *cd; + int rc; + opal_value_t *iptr; + size_t n; + + /* this is in the PMIx local thread - need to threadshift to + * our own thread as we will be accessing framework-global + * lists and objects */ + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s RECEIVED NOTIFICATION OF STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), status); + + cd = OBJ_NEW(pmix2x_threadshift_t); + + /* convert the incoming status */ + cd->status = pmix2x_convert_rc(status); + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CONVERTED STATUS %d TO STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), status, cd->status); + /* convert the nspace/rank to an opal_process_name_t */ + if (NULL == source) { + cd->pname.jobid = OPAL_NAME_INVALID->jobid; + cd->pname.vpid = OPAL_NAME_INVALID->vpid; + } else { + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&cd->pname.jobid, source->nspace))) { + OPAL_ERROR_LOG(rc); + OBJ_RELEASE(cd); + return; + } + cd->pname.vpid = source->rank; + } + + /* convert the array of info */ + if (NULL != info) { + cd->info = OBJ_NEW(opal_list_t); + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + cd->nondefault = true; + } + iptr = OBJ_NEW(opal_value_t); + iptr->key = strdup(info[n].key); + pmix2x_value_unload(iptr, &info[n].value); + opal_list_append(cd->info, &iptr->super); + } + } + /* now push it into the local thread */ + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _event_hdlr, cd); + event_active(&cd->ev, EV_WRITE, 1); + + /* we don't need any of the data they provided, + * so let them go */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } +} + +pmix_status_t pmix2x_convert_opalrc(int rc) +{ + switch (rc) { + case OPAL_ERR_DEBUGGER_RELEASE: + return PMIX_ERR_DEBUGGER_RELEASE; + + case OPAL_ERR_NOT_IMPLEMENTED: + case OPAL_ERR_NOT_SUPPORTED: + return PMIX_ERR_NOT_SUPPORTED; + + case OPAL_ERR_NOT_FOUND: + return PMIX_ERR_NOT_FOUND; + + case OPAL_ERR_PERM: + case OPAL_ERR_UNREACH: + case OPAL_ERR_SERVER_NOT_AVAIL: + return PMIX_ERR_UNREACH; + + case OPAL_ERR_BAD_PARAM: + return PMIX_ERR_BAD_PARAM; + + case OPAL_ERR_OUT_OF_RESOURCE: + return PMIX_ERR_OUT_OF_RESOURCE; + + case OPAL_ERR_DATA_VALUE_NOT_FOUND: + return PMIX_ERR_DATA_VALUE_NOT_FOUND; + + case OPAL_ERR_TIMEOUT: + return PMIX_ERR_TIMEOUT; + + case OPAL_ERR_WOULD_BLOCK: + return PMIX_ERR_WOULD_BLOCK; + + case OPAL_EXISTS: + return PMIX_EXISTS; + + case OPAL_ERROR: + return PMIX_ERROR; + case OPAL_SUCCESS: + return PMIX_SUCCESS; + default: + return PMIX_ERROR; + } +} + +int pmix2x_convert_rc(pmix_status_t rc) +{ + switch (rc) { + case PMIX_ERR_DEBUGGER_RELEASE: + return OPAL_ERR_DEBUGGER_RELEASE; + + case PMIX_ERR_NOT_SUPPORTED: + return OPAL_ERR_NOT_SUPPORTED; + + case PMIX_ERR_NOT_FOUND: + return OPAL_ERR_NOT_FOUND; + + case PMIX_ERR_OUT_OF_RESOURCE: + return OPAL_ERR_OUT_OF_RESOURCE; + + case PMIX_ERR_INIT: + return OPAL_ERROR; + + case PMIX_ERR_BAD_PARAM: + return OPAL_ERR_BAD_PARAM; + + case PMIX_ERR_UNREACH: + case PMIX_ERR_NO_PERMISSIONS: + return OPAL_ERR_UNREACH; + + case PMIX_ERR_TIMEOUT: + return OPAL_ERR_TIMEOUT; + + case PMIX_ERR_WOULD_BLOCK: + return OPAL_ERR_WOULD_BLOCK; + + case PMIX_ERR_LOST_CONNECTION_TO_SERVER: + case PMIX_ERR_LOST_PEER_CONNECTION: + case PMIX_ERR_LOST_CONNECTION_TO_CLIENT: + return OPAL_ERR_COMM_FAILURE; + + case PMIX_EXISTS: + return OPAL_EXISTS; + + case PMIX_ERROR: + return OPAL_ERROR; + case PMIX_SUCCESS: + return OPAL_SUCCESS; + default: + return OPAL_ERROR; + } +} + +pmix_scope_t pmix2x_convert_opalscope(opal_pmix_scope_t scope) { + switch(scope) { + case OPAL_PMIX_LOCAL: + return PMIX_LOCAL; + case OPAL_PMIX_REMOTE: + return PMIX_REMOTE; + case OPAL_PMIX_GLOBAL: + return PMIX_GLOBAL; + default: + return PMIX_SCOPE_UNDEF; + } +} + +pmix_data_range_t pmix2x_convert_opalrange(opal_pmix_data_range_t range) { + switch(range) { + case OPAL_PMIX_RANGE_UNDEF: + return PMIX_RANGE_UNDEF; + case OPAL_PMIX_RANGE_LOCAL: + return PMIX_RANGE_LOCAL; + case OPAL_PMIX_RANGE_NAMESPACE: + return PMIX_RANGE_NAMESPACE; + case OPAL_PMIX_RANGE_SESSION: + return PMIX_RANGE_SESSION; + case OPAL_PMIX_RANGE_GLOBAL: + return PMIX_RANGE_GLOBAL; + case OPAL_PMIX_RANGE_CUSTOM: + return PMIX_RANGE_CUSTOM; + default: + return PMIX_SCOPE_UNDEF; + } +} + +opal_pmix_data_range_t pmix2x_convert_range(pmix_data_range_t range) { + switch(range) { + case PMIX_RANGE_UNDEF: + return OPAL_PMIX_RANGE_UNDEF; + case PMIX_RANGE_LOCAL: + return OPAL_PMIX_RANGE_LOCAL; + case PMIX_RANGE_NAMESPACE: + return OPAL_PMIX_RANGE_NAMESPACE; + case PMIX_RANGE_SESSION: + return OPAL_PMIX_RANGE_SESSION; + case PMIX_RANGE_GLOBAL: + return OPAL_PMIX_RANGE_GLOBAL; + case PMIX_RANGE_CUSTOM: + return OPAL_PMIX_RANGE_CUSTOM; + default: + return OPAL_PMIX_SCOPE_UNDEF; + } +} + + +void pmix2x_value_load(pmix_value_t *v, + opal_value_t *kv) +{ + switch(kv->type) { + case OPAL_UNDEF: + v->type = PMIX_UNDEF; + break; + case OPAL_BOOL: + v->type = PMIX_BOOL; + memcpy(&(v->data.flag), &kv->data.flag, 1); + break; + case OPAL_BYTE: + v->type = PMIX_BYTE; + memcpy(&(v->data.byte), &kv->data.byte, 1); + break; + case OPAL_STRING: + v->type = PMIX_STRING; + if (NULL != kv->data.string) { + v->data.string = strdup(kv->data.string); + } else { + v->data.string = NULL; + } + break; + case OPAL_SIZE: + v->type = PMIX_SIZE; + v->data.size = (size_t)kv->data.size; + break; + case OPAL_PID: + v->type = PMIX_PID; + memcpy(&(v->data.pid), &kv->data.pid, sizeof(pid_t)); + break; + case OPAL_INT: + v->type = PMIX_INT; + memcpy(&(v->data.integer), &kv->data.integer, sizeof(int)); + break; + case OPAL_INT8: + v->type = PMIX_INT8; + memcpy(&(v->data.int8), &kv->data.int8, 1); + break; + case OPAL_INT16: + v->type = PMIX_INT16; + memcpy(&(v->data.int16), &kv->data.int16, 2); + break; + case OPAL_INT32: + v->type = PMIX_INT32; + memcpy(&(v->data.int32), &kv->data.int32, 4); + break; + case OPAL_INT64: + v->type = PMIX_INT64; + memcpy(&(v->data.int64), &kv->data.int64, 8); + break; + case OPAL_UINT: + v->type = PMIX_UINT; + memcpy(&(v->data.uint), &kv->data.uint, sizeof(int)); + break; + case OPAL_UINT8: + v->type = PMIX_UINT8; + memcpy(&(v->data.uint8), &kv->data.uint8, 1); + break; + case OPAL_UINT16: + v->type = PMIX_UINT16; + memcpy(&(v->data.uint16), &kv->data.uint16, 2); + break; + case OPAL_UINT32: + v->type = PMIX_UINT32; + memcpy(&(v->data.uint32), &kv->data.uint32, 4); + break; + case OPAL_UINT64: + v->type = PMIX_UINT64; + memcpy(&(v->data.uint64), &kv->data.uint64, 8); + break; + case OPAL_FLOAT: + v->type = PMIX_FLOAT; + memcpy(&(v->data.fval), &kv->data.fval, sizeof(float)); + break; + case OPAL_DOUBLE: + v->type = PMIX_DOUBLE; + memcpy(&(v->data.dval), &kv->data.dval, sizeof(double)); + break; + case OPAL_TIMEVAL: + v->type = PMIX_TIMEVAL; + memcpy(&(v->data.tv), &kv->data.tv, sizeof(struct timeval)); + break; + case OPAL_BYTE_OBJECT: + v->type = PMIX_BYTE_OBJECT; + if (NULL != kv->data.bo.bytes) { + v->data.bo.bytes = (char*)malloc(kv->data.bo.size); + memcpy(v->data.bo.bytes, kv->data.bo.bytes, kv->data.bo.size); + v->data.bo.size = (size_t)kv->data.bo.size; + } else { + v->data.bo.bytes = NULL; + v->data.bo.size = 0; + } + break; + default: + /* silence warnings */ + break; + } +} + +int pmix2x_value_unload(opal_value_t *kv, + const pmix_value_t *v) +{ + int rc=OPAL_SUCCESS; + + + switch(v->type) { + case PMIX_UNDEF: + rc = OPAL_ERR_UNKNOWN_DATA_TYPE; + break; + case PMIX_BOOL: + kv->type = OPAL_BOOL; + memcpy(&kv->data.flag, &(v->data.flag), 1); + break; + case PMIX_BYTE: + kv->type = OPAL_BYTE; + memcpy(&kv->data.byte, &(v->data.byte), 1); + break; + case PMIX_STRING: + kv->type = OPAL_STRING; + if (NULL != v->data.string) { + kv->data.string = strdup(v->data.string); + } + break; + case PMIX_SIZE: + kv->type = OPAL_SIZE; + kv->data.size = (int)v->data.size; + break; + case PMIX_PID: + kv->type = OPAL_PID; + memcpy(&kv->data.pid, &(v->data.pid), sizeof(pid_t)); + break; + case PMIX_INT: + kv->type = OPAL_INT; + memcpy(&kv->data.integer, &(v->data.integer), sizeof(int)); + break; + case PMIX_INT8: + kv->type = OPAL_INT8; + memcpy(&kv->data.int8, &(v->data.int8), 1); + break; + case PMIX_INT16: + kv->type = OPAL_INT16; + memcpy(&kv->data.int16, &(v->data.int16), 2); + break; + case PMIX_INT32: + kv->type = OPAL_INT32; + memcpy(&kv->data.int32, &(v->data.int32), 4); + break; + case PMIX_INT64: + kv->type = OPAL_INT64; + memcpy(&kv->data, &(v->data.int64), 8); + break; + case PMIX_UINT: + kv->type = OPAL_UINT; + memcpy(&kv->data, &(v->data.uint), sizeof(int)); + break; + case PMIX_UINT8: + kv->type = OPAL_UINT8; + memcpy(&kv->data, &(v->data.uint8), 1); + break; + case PMIX_UINT16: + kv->type = OPAL_UINT16; + memcpy(&kv->data, &(v->data.uint16), 2); + break; + case PMIX_UINT32: + kv->type = OPAL_UINT32; + memcpy(&kv->data, &(v->data.uint32), 4); + break; + case PMIX_UINT64: + kv->type = OPAL_UINT64; + memcpy(&kv->data, &(v->data.uint64), 8); + break; + case PMIX_FLOAT: + kv->type = OPAL_FLOAT; + memcpy(&kv->data, &(v->data.fval), sizeof(float)); + break; + case PMIX_DOUBLE: + kv->type = OPAL_DOUBLE; + memcpy(&kv->data, &(v->data.dval), sizeof(double)); + break; + case PMIX_TIMEVAL: + kv->type = OPAL_TIMEVAL; + memcpy(&kv->data, &(v->data.tv), sizeof(struct timeval)); + break; + case PMIX_BYTE_OBJECT: + kv->type = OPAL_BYTE_OBJECT; + if (NULL != v->data.bo.bytes && 0 < v->data.bo.size) { + kv->data.bo.bytes = (uint8_t*)malloc(v->data.bo.size); + memcpy(kv->data.bo.bytes, v->data.bo.bytes, v->data.bo.size); + kv->data.bo.size = (int)v->data.bo.size; + } else { + kv->data.bo.bytes = NULL; + kv->data.bo.size = 0; + } + break; + default: + /* silence warnings */ + rc = OPAL_ERROR; + break; + } + return rc; +} + +static void _reg_hdlr(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + opal_pmix2x_event_chain_t *chain; + opal_pmix2x_single_event_t *sing = NULL; + opal_pmix2x_multi_event_t *multi = NULL; + opal_pmix2x_default_event_t *def = NULL; + opal_value_t *kv; + int i; + bool prepend = false; + size_t n; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s REGISTER HANDLER CODES %s", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + (NULL == cd->event_codes) ? "NULL" : "NON-NULL"); + + if (NULL != cd->info) { + OPAL_LIST_FOREACH(kv, cd->info, opal_value_t) { + if (0 == strcmp(kv->key, OPAL_PMIX_EVENT_ORDER_PREPEND)) { + prepend = true; + break; + } + } + } + + if (NULL == cd->event_codes) { + /* this is a default handler */ + def = OBJ_NEW(opal_pmix2x_default_event_t); + def->handler = cd->evhandler; + def->index = mca_pmix_pmix2x_component.evindex; + if (prepend) { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s PREPENDING TO DEFAULT EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_prepend(&mca_pmix_pmix2x_component.default_events, &def->super); + } else { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s APPENDING TO DEFAULT EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_append(&mca_pmix_pmix2x_component.default_events, &def->super); + } + } else if (1 == opal_list_get_size(cd->event_codes)) { + /* single handler */ + sing = OBJ_NEW(opal_pmix2x_single_event_t); + kv = (opal_value_t*)opal_list_get_first(cd->event_codes); + sing->code = kv->data.integer; + sing->index = mca_pmix_pmix2x_component.evindex; + sing->handler = cd->evhandler; + if (prepend) { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s PREPENDING TO SINGLE EVENTS WITH CODE %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), sing->code); + opal_list_prepend(&mca_pmix_pmix2x_component.single_events, &sing->super); + } else { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s APPENDING TO SINGLE EVENTS WITH CODE %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), sing->code); + opal_list_append(&mca_pmix_pmix2x_component.single_events, &sing->super); + } + } else { + multi = OBJ_NEW(opal_pmix2x_multi_event_t); + multi->ncodes = opal_list_get_size(cd->event_codes); + multi->codes = (int*)malloc(multi->ncodes * sizeof(int)); + i=0; + OPAL_LIST_FOREACH(kv, cd->event_codes, opal_value_t) { + multi->codes[i] = kv->data.integer; + ++i; + } + multi->index = mca_pmix_pmix2x_component.evindex; + multi->handler = cd->evhandler; + if (prepend) { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s PREPENDING TO MULTI EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_prepend(&mca_pmix_pmix2x_component.multi_events, &multi->super); + } else { + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s APPENDING TO MULTI EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + opal_list_append(&mca_pmix_pmix2x_component.multi_events, &multi->super); + } + } + + /* release the caller */ + if (NULL != cd->cbfunc) { + cd->cbfunc(OPAL_SUCCESS, mca_pmix_pmix2x_component.evindex, cd->cbdata); + } + mca_pmix_pmix2x_component.evindex++; + + /* check if any matching notifications have been cached - only nondefault + * events will have been cached*/ + if (NULL == def) { + /* check single code registrations */ + if (NULL != sing) { + OPAL_LIST_FOREACH(chain, &mca_pmix_pmix2x_component.cache, opal_pmix2x_event_chain_t) { + if (sing->code == chain->status) { + opal_list_remove_item(&mca_pmix_pmix2x_component.cache, &chain->super); + chain->sing = sing; + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + } else if (NULL != multi) { + /* check for multi code registrations */ + OPAL_LIST_FOREACH(chain, &mca_pmix_pmix2x_component.cache, opal_pmix2x_event_chain_t) { + for (n=0; n < multi->ncodes; n++) { + if (multi->codes[n] == chain->status) { + opal_list_remove_item(&mca_pmix_pmix2x_component.cache, &chain->super); + chain->multi = multi; + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + } + } + } + + OBJ_RELEASE(cd); + return; +} +static void register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, + void *cbdata) +{ + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + + OPAL_PMIX_THREADSHIFT(event_codes, info, evhandler, _reg_hdlr, cbfunc, cbdata); + return; +} + +static void _dereg_hdlr(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + opal_pmix2x_single_event_t *sing; + opal_pmix2x_multi_event_t *multi; + opal_pmix2x_default_event_t *def; + + /* check the single events first */ + OPAL_LIST_FOREACH(sing, &mca_pmix_pmix2x_component.single_events, opal_pmix2x_single_event_t) { + if (cd->handler == sing->index) { + opal_list_remove_item(&mca_pmix_pmix2x_component.single_events, &sing->super); + OBJ_RELEASE(sing); + goto release; + } + } + /* check multi events */ + OPAL_LIST_FOREACH(multi, &mca_pmix_pmix2x_component.multi_events, opal_pmix2x_multi_event_t) { + if (cd->handler == multi->index) { + opal_list_remove_item(&mca_pmix_pmix2x_component.multi_events, &multi->super); + OBJ_RELEASE(multi); + goto release; + } + } + /* check default events */ + OPAL_LIST_FOREACH(def, &mca_pmix_pmix2x_component.default_events, opal_pmix2x_default_event_t) { + if (cd->handler == def->index) { + opal_list_remove_item(&mca_pmix_pmix2x_component.default_events, &def->super); + OBJ_RELEASE(def); + break; + } + } + + release: + if (NULL != cd->opcbfunc) { + cd->opcbfunc(OPAL_SUCCESS, cd->cbdata); + } + OBJ_RELEASE(cd); +} + +static void deregister_handler(size_t evhandler, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + OPAL_PMIX_OP_THREADSHIFT(evhandler, _dereg_hdlr, cbfunc, cbdata); + return; +} + +static void _notify_event(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + size_t i; + opal_pmix2x_single_event_t *sing; + opal_pmix2x_multi_event_t *multi; + opal_pmix2x_default_event_t *def; + opal_pmix2x_event_chain_t *chain; + + /* check the single events first */ + OPAL_LIST_FOREACH(sing, &mca_pmix_pmix2x_component.single_events, opal_pmix2x_single_event_t) { + if (cd->status == sing->code) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + chain = OBJ_NEW(opal_pmix2x_event_chain_t); + chain->status = cd->status; + chain->range = pmix2x_convert_opalrange(cd->range); + chain->source = *(cd->source); + chain->info = cd->info; + chain->final_cbfunc = cd->opcbfunc; + chain->final_cbdata = cd->cbdata; + chain->sing = sing; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "[%s] CALLING SINGLE EVHDLR FOR STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), chain->status); + sing->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + /* check multi events */ + OPAL_LIST_FOREACH(multi, &mca_pmix_pmix2x_component.multi_events, opal_pmix2x_multi_event_t) { + for (i=0; i < multi->ncodes; i++) { + if (cd->status == multi->codes[i]) { + /* found it - invoke the handler, pointing its + * callback function to our progression function */ + chain = OBJ_NEW(opal_pmix2x_event_chain_t); + chain->status = cd->status; + chain->range = pmix2x_convert_opalrange(cd->range); + chain->source = *(cd->source); + chain->info = cd->info; + chain->final_cbfunc = cd->opcbfunc; + chain->final_cbdata = cd->cbdata; + chain->multi = multi; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "[%s] CALLING MULTI EVHDLR FOR STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), chain->status); + multi->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + } + } + /* check default events */ + if (0 < opal_list_get_size(&mca_pmix_pmix2x_component.default_events)) { + def = (opal_pmix2x_default_event_t*)opal_list_get_first(&mca_pmix_pmix2x_component.default_events); + chain = OBJ_NEW(opal_pmix2x_event_chain_t); + chain->status = cd->status; + chain->range = pmix2x_convert_opalrange(cd->range); + chain->source = *(cd->source); + chain->info = cd->info; + chain->final_cbfunc = cd->opcbfunc; + chain->final_cbdata = cd->cbdata; + chain->def = def; + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "[%s] CALLING DEFAULT EVHDLR FOR STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), chain->status); + def->handler(chain->status, &chain->source, + chain->info, &chain->results, + progress_local_event_hdlr, (void*)chain); + OBJ_RELEASE(cd); + return; + } + + /* if we get here, then there are no registered event handlers */ + if (NULL != cd->opcbfunc) { + cd->opcbfunc(OPAL_ERR_NOT_FOUND, cd->cbdata); + } + OBJ_RELEASE(cd); + return; +} + +static int notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + OPAL_PMIX_NOTIFY_THREADSHIFT(status, source, range, info, _notify_event, cbfunc, cbdata); + return OPAL_SUCCESS; +} + +/**** INSTANTIATE INTERNAL CLASSES ****/ +OBJ_CLASS_INSTANCE(opal_pmix2x_jobid_trkr_t, + opal_list_item_t, + NULL, NULL); + +OBJ_CLASS_INSTANCE(opal_pmix2x_single_event_t, + opal_list_item_t, + NULL, NULL); + +static void mtevcon(opal_pmix2x_multi_event_t *p) +{ + p->codes = NULL; + p->ncodes = 0; +} +static void mtevdes(opal_pmix2x_multi_event_t *p) +{ + if (NULL != p->codes) { + free(p->codes); + } +} +OBJ_CLASS_INSTANCE(opal_pmix2x_multi_event_t, + opal_list_item_t, + mtevcon, mtevdes); + +OBJ_CLASS_INSTANCE(opal_pmix2x_default_event_t, + opal_list_item_t, + NULL, NULL); + +static void chcon(opal_pmix2x_event_chain_t *p) +{ + p->nondefault = false; + p->info = NULL; + OBJ_CONSTRUCT(&p->results, opal_list_t); + p->sing = NULL; + p->multi = NULL; + p->def = NULL; + p->final_cbfunc = NULL; + p->final_cbdata = NULL; +} +static void chdes(opal_pmix2x_event_chain_t *p) +{ + OPAL_LIST_DESTRUCT(&p->results); +} +OBJ_CLASS_INSTANCE(opal_pmix2x_event_chain_t, + opal_list_item_t, + chcon, chdes); + +static void opcon(pmix2x_opcaddy_t *p) +{ + memset(&p->p, 0, sizeof(pmix_proc_t)); + p->procs = NULL; + p->nprocs = 0; + p->error_procs = NULL; + p->nerror_procs = 0; + p->info = NULL; + p->ninfo = 0; + p->apps = NULL; + p->sz = 0; + p->active = false; + p->opcbfunc = NULL; + p->mdxcbfunc = NULL; + p->valcbfunc = NULL; + p->lkcbfunc = NULL; + p->spcbfunc = NULL; + p->cbdata = NULL; +} +static void opdes(pmix2x_opcaddy_t *p) +{ + if (NULL != p->procs) { + PMIX_PROC_FREE(p->procs, p->nprocs); + } + if (NULL != p->error_procs) { + PMIX_PROC_FREE(p->error_procs, p->nerror_procs); + } + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->sz); + } + if (NULL != p->apps) { + PMIX_APP_FREE(p->apps, p->sz); + } +} +OBJ_CLASS_INSTANCE(pmix2x_opcaddy_t, + opal_object_t, + opcon, opdes); + +static void ocadcon(pmix2x_opalcaddy_t *p) +{ + OBJ_CONSTRUCT(&p->procs, opal_list_t); + OBJ_CONSTRUCT(&p->info, opal_list_t); + OBJ_CONSTRUCT(&p->apps, opal_list_t); + p->opcbfunc = NULL; + p->dmdxfunc = NULL; + p->mdxcbfunc = NULL; + p->lkupcbfunc = NULL; + p->spwncbfunc = NULL; + p->cbdata = NULL; + p->odmdxfunc = NULL; + p->ocbdata = NULL; +} +static void ocaddes(pmix2x_opalcaddy_t *p) +{ + OPAL_LIST_DESTRUCT(&p->procs); + OPAL_LIST_DESTRUCT(&p->info); + OPAL_LIST_DESTRUCT(&p->apps); +} +OBJ_CLASS_INSTANCE(pmix2x_opalcaddy_t, + opal_object_t, + ocadcon, ocaddes); + +static void tscon(pmix2x_threadshift_t *p) +{ + p->active = false; + p->source = NULL; + p->event_codes = NULL; + p->info = NULL; + p->evhandler = NULL; + p->cbfunc = NULL; + p->opcbfunc = NULL; + p->cbdata = NULL; +} +OBJ_CLASS_INSTANCE(pmix2x_threadshift_t, + opal_object_t, + tscon, NULL); diff --git a/opal/mca/pmix/pmix2x/pmix2x.h b/opal/mca/pmix/pmix2x/pmix2x.h new file mode 100644 index 0000000000..c0da822468 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix2x.h @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_PMIX_PMIX2X_H +#define MCA_PMIX_PMIX2X_H + +#include "opal_config.h" + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif + +#include "opal/class/opal_list.h" +#include "opal/mca/mca.h" +#include "opal/mca/event/event.h" +#include "opal/util/proc.h" + +#include "opal/mca/pmix/pmix.h" +#include "pmix_server.h" +#include "pmix/pmix_common.h" + +BEGIN_C_DECLS + +typedef struct { + opal_pmix_base_component_t super; + opal_list_t jobids; + bool native_launch; + size_t evindex; + opal_list_t single_events; + opal_list_t multi_events; + opal_list_t default_events; + int cache_size; + opal_list_t cache; +} mca_pmix_pmix2x_component_t; + +OPAL_DECLSPEC extern mca_pmix_pmix2x_component_t mca_pmix_pmix2x_component; + +OPAL_DECLSPEC extern const opal_pmix_base_module_t opal_pmix_pmix2x_module; + +/**** INTERNAL OBJECTS ****/ +typedef struct { + opal_list_item_t super; + opal_jobid_t jobid; + char nspace[PMIX_MAX_NSLEN + 1]; +} opal_pmix2x_jobid_trkr_t; +OBJ_CLASS_DECLARATION(opal_pmix2x_jobid_trkr_t); + +typedef struct { + opal_list_item_t super; + size_t index; + int code; + opal_pmix_notification_fn_t handler; +} opal_pmix2x_single_event_t; +OBJ_CLASS_DECLARATION(opal_pmix2x_single_event_t); + +typedef struct { + opal_list_item_t super; + size_t index; + int *codes; + size_t ncodes; + opal_pmix_notification_fn_t handler; +} opal_pmix2x_multi_event_t; +OBJ_CLASS_DECLARATION(opal_pmix2x_multi_event_t); + +typedef struct { + opal_list_item_t super; + size_t index; + opal_pmix_notification_fn_t handler; +} opal_pmix2x_default_event_t; +OBJ_CLASS_DECLARATION(opal_pmix2x_default_event_t); + +typedef struct { + opal_list_item_t super; + int status; + bool nondefault; + opal_process_name_t source; + pmix_data_range_t range; + opal_list_t *info; + opal_list_t results; + opal_pmix2x_single_event_t *sing; + opal_pmix2x_multi_event_t *multi; + opal_pmix2x_default_event_t *def; + opal_pmix_op_cbfunc_t final_cbfunc; + void *final_cbdata; +} opal_pmix2x_event_chain_t; +OBJ_CLASS_DECLARATION(opal_pmix2x_event_chain_t); + +typedef struct { + opal_object_t super; + pmix_status_t status; + pmix_proc_t p; + pmix_proc_t *procs; + size_t nprocs; + pmix_proc_t *error_procs; + size_t nerror_procs; + pmix_info_t *info; + size_t ninfo; + pmix_app_t *apps; + size_t sz; + volatile bool active; + opal_pmix_op_cbfunc_t opcbfunc; + opal_pmix_modex_cbfunc_t mdxcbfunc; + opal_pmix_value_cbfunc_t valcbfunc; + opal_pmix_lookup_cbfunc_t lkcbfunc; + opal_pmix_spawn_cbfunc_t spcbfunc; + void *cbdata; +} pmix2x_opcaddy_t; +OBJ_CLASS_DECLARATION(pmix2x_opcaddy_t); + +typedef struct { + opal_object_t super; + opal_list_t procs; + opal_list_t info; + opal_list_t apps; + pmix_op_cbfunc_t opcbfunc; + pmix_dmodex_response_fn_t dmdxfunc; + pmix_modex_cbfunc_t mdxcbfunc; + pmix_lookup_cbfunc_t lkupcbfunc; + pmix_spawn_cbfunc_t spwncbfunc; + void *cbdata; + opal_pmix_release_cbfunc_t odmdxfunc; + void *ocbdata; +} pmix2x_opalcaddy_t; +OBJ_CLASS_DECLARATION(pmix2x_opalcaddy_t); + +typedef struct { + opal_object_t super; + opal_event_t ev; + volatile bool active; + size_t id; + int status; + opal_process_name_t pname; + opal_jobid_t jobid; + const opal_process_name_t *source; + opal_pmix_data_range_t range; + bool nondefault; + size_t handler; + opal_list_t *event_codes; + opal_list_t *info; + opal_pmix_notification_fn_t evhandler; + opal_pmix_evhandler_reg_cbfunc_t cbfunc; + opal_pmix_op_cbfunc_t opcbfunc; + void *cbdata; +} pmix2x_threadshift_t; +OBJ_CLASS_DECLARATION(pmix2x_threadshift_t); + +#define OPAL_PMIX_OPCD_THREADSHIFT(i, s, sr, if, nif, fn, cb, cd) \ + do { \ + pmix2x_opalcaddy_t *_cd; \ + _cd = OBJ_NEW(pmix2x_opalcaddy_t); \ + _cd->id = (i); \ + _cd->status = (s); \ + _cd->source = (sr); \ + _cd->info = (i); \ + _cd->evcbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_OP_THREADSHIFT(e, fn, cb, cd) \ + do { \ + pmix2x_threadshift_t *_cd; \ + _cd = OBJ_NEW(pmix2x_threadshift_t); \ + _cd->handler = (e); \ + _cd->opcbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_THREADSHIFT(e, i, eh, fn, cb, cd) \ + do { \ + pmix2x_threadshift_t *_cd; \ + _cd = OBJ_NEW(pmix2x_threadshift_t); \ + _cd->event_codes = (e); \ + _cd->info = (i); \ + _cd->evhandler = (eh); \ + _cd->cbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_NOTIFY_THREADSHIFT(s, sr, r, i, fn, cb, cd) \ + do { \ + pmix2x_threadshift_t *_cd; \ + _cd = OBJ_NEW(pmix2x_threadshift_t); \ + _cd->status = (s); \ + _cd->source = (sr); \ + _cd->range = (r); \ + _cd->info = (i); \ + _cd->opcbfunc = (cb); \ + _cd->cbdata = (cd); \ + event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +/**** CLIENT FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int pmix2x_client_init(void); +OPAL_MODULE_DECLSPEC int pmix2x_client_finalize(void); +OPAL_MODULE_DECLSPEC int pmix2x_initialized(void); +OPAL_MODULE_DECLSPEC int pmix2x_abort(int flag, const char *msg, + opal_list_t *procs); +OPAL_MODULE_DECLSPEC int pmix2x_commit(void); +OPAL_MODULE_DECLSPEC int pmix2x_fence(opal_list_t *procs, int collect_data); +OPAL_MODULE_DECLSPEC int pmix2x_fencenb(opal_list_t *procs, int collect_data, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_put(opal_pmix_scope_t scope, + opal_value_t *val); +OPAL_MODULE_DECLSPEC int pmix2x_get(const opal_process_name_t *proc, const char *key, + opal_list_t *info, opal_value_t **val); +OPAL_MODULE_DECLSPEC int pmix2x_getnb(const opal_process_name_t *proc, const char *key, + opal_list_t *info, + opal_pmix_value_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_publish(opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix2x_publishnb(opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_lookup(opal_list_t *data, opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix2x_lookupnb(char **keys, opal_list_t *info, + opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_unpublish(char **keys, opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix2x_unpublishnb(char **keys, opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid); +OPAL_MODULE_DECLSPEC int pmix2x_spawnnb(opal_list_t *job_info, opal_list_t *apps, + opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_connect(opal_list_t *procs); +OPAL_MODULE_DECLSPEC int pmix2x_connectnb(opal_list_t *procs, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_disconnect(opal_list_t *procs); +OPAL_MODULE_DECLSPEC int pmix2x_disconnectnb(opal_list_t *procs, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_resolve_peers(const char *nodename, opal_jobid_t jobid, + opal_list_t *procs); +OPAL_MODULE_DECLSPEC int pmix2x_resolve_nodes(opal_jobid_t jobid, char **nodelist); + +/**** COMMON FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int pmix2x_store_local(const opal_process_name_t *proc, + opal_value_t *val); + +/**** SERVER SOUTHBOUND FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int pmix2x_server_init(opal_pmix_server_module_t *module, + opal_list_t *info); +OPAL_MODULE_DECLSPEC int pmix2x_server_finalize(void); +OPAL_MODULE_DECLSPEC int pmix2x_server_gen_regex(const char *input, char **regex); +OPAL_MODULE_DECLSPEC int pmix2x_server_gen_ppn(const char *input, char **ppn); +OPAL_MODULE_DECLSPEC int pmix2x_server_register_nspace(opal_jobid_t jobid, + int nlocalprocs, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC void pmix2x_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_server_register_client(const opal_process_name_t *proc, + uid_t uid, gid_t gid, + void *server_object, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC void pmix2x_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_server_setup_fork(const opal_process_name_t *proc, char ***env); +OPAL_MODULE_DECLSPEC int pmix2x_server_dmodex(const opal_process_name_t *proc, + opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int pmix2x_server_notify_event(int status, + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); + + +/**** COMPONENT UTILITY FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC void pmix2x_event_hdlr(size_t evhdlr_registration_id, + pmix_status_t status, const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC pmix_status_t pmix2x_convert_opalrc(int rc); +OPAL_MODULE_DECLSPEC int pmix2x_convert_rc(pmix_status_t rc); +OPAL_MODULE_DECLSPEC pmix_scope_t pmix2x_convert_opalscope(opal_pmix_scope_t scope); +OPAL_MODULE_DECLSPEC pmix_data_range_t pmix2x_convert_opalrange(opal_pmix_data_range_t range); +OPAL_MODULE_DECLSPEC opal_pmix_data_range_t pmix2x_convert_range(pmix_data_range_t range); +OPAL_MODULE_DECLSPEC void pmix2x_value_load(pmix_value_t *v, + opal_value_t *kv); +OPAL_MODULE_DECLSPEC int pmix2x_value_unload(opal_value_t *kv, + const pmix_value_t *v); + +END_C_DECLS + +#endif /* MCA_PMIX_EXTERNAL_H */ diff --git a/opal/mca/pmix/pmix114/pmix1_client.c b/opal/mca/pmix/pmix2x/pmix2x_client.c similarity index 75% rename from opal/mca/pmix/pmix114/pmix1_client.c rename to opal/mca/pmix/pmix2x/pmix2x_client.c index 6ce9fcc1dd..e2827db11b 100644 --- a/opal/mca/pmix/pmix114/pmix1_client.c +++ b/opal/mca/pmix/pmix2x/pmix2x_client.c @@ -1,9 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2014 Mellanox Technologies, Inc. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -28,75 +28,37 @@ #include "opal/util/proc.h" #include "opal/mca/pmix/base/base.h" -#include "pmix1.h" -#include "opal/mca/pmix/pmix114/pmix/include/pmix.h" -#include "opal/mca/pmix/pmix114/pmix/src/buffer_ops/buffer_ops.h" +#include "pmix2x.h" +#include "pmix.h" static pmix_proc_t my_proc; static char *dbgvalue=NULL; -static int errhdler_ref = 0; +static size_t errhdler_ref = 0; -static void release_cbfunc(void *cbdata) -{ - pmix1_opalcaddy_t *cd = (pmix1_opalcaddy_t*)cbdata; - OBJ_RELEASE(cd); -} -static void myerr(pmix_status_t status, - pmix_proc_t procs[], size_t nprocs, - pmix_info_t info[], size_t ninfo) -{ - int rc; - opal_namelist_t *nm; - opal_value_t *iptr; - size_t n; - pmix1_opalcaddy_t *cd; +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + } while (0) - /* convert the incoming status */ - rc = pmix1_convert_rc(status); - - /* setup the caddy */ - cd = OBJ_NEW(pmix1_opalcaddy_t); - - /* convert the array of procs */ - for (n=0; n < nprocs; n++) { - nm = OBJ_NEW(opal_namelist_t); - if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { - OPAL_ERROR_LOG(rc); - OBJ_RELEASE(cd); - return; - } - nm->name.vpid = procs[n].rank; - opal_list_append(&cd->procs, &nm->super); - } - - /* convert the array of info */ - for (n=0; n < ninfo; n++) { - iptr = OBJ_NEW(opal_value_t); - iptr->key = strdup(info[n].key); - pmix1_value_unload(iptr, &info[n].value); - opal_list_append(&cd->info, &iptr->super); - } - - /* call the base errhandler */ - opal_pmix_base_errhandler(rc, &cd->procs, &cd->info, release_cbfunc, cd); -} static void errreg_cbfunc (pmix_status_t status, - int errhandler_ref, + size_t errhandler_ref, void *cbdata) { errhdler_ref = errhandler_ref; opal_output_verbose(5, opal_pmix_base_framework.framework_output, - "PMIX client errreg_cbfunc - error handler registered status=%d, reference=%d", - status, errhandler_ref); + "PMIX client errreg_cbfunc - error handler registered status=%d, reference=%lu", + status, (unsigned long)errhandler_ref); } -int pmix1_client_init(void) +int pmix2x_client_init(void) { opal_process_name_t pname; pmix_status_t rc; int dbg; - opal_pmix1_jobid_trkr_t *job; + opal_pmix2x_jobid_trkr_t *job; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client init"); @@ -105,56 +67,54 @@ int pmix1_client_init(void) asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg); putenv(dbgvalue); } - rc = PMIx_Init(&my_proc); + + rc = PMIx_Init(&my_proc, NULL, 0); if (PMIX_SUCCESS != rc) { - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } /* store our jobid and rank */ if (NULL != getenv(OPAL_MCA_PREFIX"orte_launch")) { /* if we were launched by the OMPI RTE, then * the jobid is in a special format - so get it */ - mca_pmix_pmix114_component.native_launch = true; + mca_pmix_pmix2x_component.native_launch = true; opal_convert_string_to_jobid(&pname.jobid, my_proc.nspace); } else { /* we were launched by someone else, so make the * jobid just be the hash of the nspace */ OPAL_HASH_STR(my_proc.nspace, pname.jobid); - /* keep it from being negative */ - pname.jobid &= ~(0x8000); } /* insert this into our list of jobids - it will be the * first, and so we'll check it first */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); (void)strncpy(job->nspace, my_proc.nspace, PMIX_MAX_NSLEN); job->jobid = pname.jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); pname.vpid = my_proc.rank; opal_proc_set_name(&pname); - /* register the errhandler */ - PMIx_Register_errhandler(NULL, 0, myerr, errreg_cbfunc, NULL ); + /* register the default event handler */ + PMIx_Register_event_handler(NULL, 0, NULL, 0, pmix2x_event_hdlr, errreg_cbfunc, NULL); return OPAL_SUCCESS; } -int pmix1_client_finalize(void) +int pmix2x_client_finalize(void) { pmix_status_t rc; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client finalize"); - /* deregister the errhandler */ - PMIx_Deregister_errhandler(errhdler_ref, NULL, NULL); + /* deregister the default event handler */ + PMIx_Deregister_event_handler(errhdler_ref, NULL, NULL); - rc = PMIx_Finalize(); - - return pmix1_convert_rc(rc); + rc = PMIx_Finalize(NULL, 0); + return pmix2x_convert_rc(rc); } -int pmix1_initialized(void) +int pmix2x_initialized(void) { opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client initialized"); @@ -162,14 +122,14 @@ int pmix1_initialized(void) return PMIx_Initialized(); } -int pmix1_abort(int flag, const char *msg, +int pmix2x_abort(int flag, const char *msg, opal_list_t *procs) { pmix_status_t rc; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client abort"); @@ -183,14 +143,13 @@ int pmix1_abort(int flag, const char *msg, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; } } if (NULL == job) { - free(parray); return OPAL_ERR_NOT_FOUND; } (void)strncpy(parray[n].nspace, job->nspace, PMIX_MAX_NSLEN); @@ -205,21 +164,24 @@ int pmix1_abort(int flag, const char *msg, /* release the array */ PMIX_PROC_FREE(parray, cnt); - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } -int pmix1_store_local(const opal_process_name_t *proc, opal_value_t *val) +int pmix2x_store_local(const opal_process_name_t *proc, opal_value_t *val) { pmix_value_t kv; pmix_status_t rc; pmix_proc_t p; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ if (NULL != proc) { /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == proc->jobid) { job = jptr; break; @@ -238,40 +200,40 @@ int pmix1_store_local(const opal_process_name_t *proc, opal_value_t *val) } PMIX_VALUE_CONSTRUCT(&kv); - pmix1_value_load(&kv, val); + pmix2x_value_load(&kv, val); rc = PMIx_Store_internal(&p, val->key, &kv); PMIX_VALUE_DESTRUCT(&kv); - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } -int pmix1_commit(void) +int pmix2x_commit(void) { pmix_status_t rc; rc = PMIx_Commit(); - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } static void opcbfunc(pmix_status_t status, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata; if (NULL != op->opcbfunc) { - op->opcbfunc(pmix1_convert_rc(status), op->cbdata); + op->opcbfunc(pmix2x_convert_rc(status), op->cbdata); } OBJ_RELEASE(op); } -int pmix1_fence(opal_list_t *procs, int collect_data) +int pmix2x_fence(opal_list_t *procs, int collect_data) { pmix_status_t rc; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; pmix_info_t info, *iptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client fence"); @@ -285,14 +247,13 @@ int pmix1_fence(opal_list_t *procs, int collect_data) /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; } } if (NULL == job) { - free(parray); return OPAL_ERR_NOT_FOUND; } (void)strncpy(parray[n].nspace, job->nspace, PMIX_MAX_NSLEN); @@ -321,20 +282,23 @@ int pmix1_fence(opal_list_t *procs, int collect_data) PMIX_INFO_DESTRUCT(&info); } - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } -int pmix1_fencenb(opal_list_t *procs, int collect_data, +int pmix2x_fencenb(opal_list_t *procs, int collect_data, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t rc; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - pmix1_opcaddy_t *op; + pmix2x_opcaddy_t *op; pmix_info_t info, *iptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client fence_nb"); @@ -348,14 +312,13 @@ int pmix1_fencenb(opal_list_t *procs, int collect_data, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; } } if (NULL == job) { - free(parray); return OPAL_ERR_NOT_FOUND; } (void)strncpy(parray[n].nspace, job->nspace, PMIX_MAX_NSLEN); @@ -377,7 +340,7 @@ int pmix1_fencenb(opal_list_t *procs, int collect_data, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; op->procs = parray; @@ -389,29 +352,29 @@ int pmix1_fencenb(opal_list_t *procs, int collect_data, OBJ_RELEASE(op); } - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } -int pmix1_put(opal_pmix_scope_t opal_scope, +int pmix2x_put(opal_pmix_scope_t opal_scope, opal_value_t *val) { pmix_value_t kv; - pmix_scope_t pmix_scope = pmix1_convert_opalscope(opal_scope); + pmix_scope_t pmix_scope = pmix2x_convert_opalscope(opal_scope); pmix_status_t rc; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client put"); PMIX_VALUE_CONSTRUCT(&kv); - pmix1_value_load(&kv, val); + pmix2x_value_load(&kv, val); rc = PMIx_Put(pmix_scope, val->key, &kv); PMIX_VALUE_DESTRUCT(&kv); - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } -int pmix1_get(const opal_process_name_t *proc, const char *key, +int pmix2x_get(const opal_process_name_t *proc, const char *key, opal_list_t *info, opal_value_t **val) { int ret; @@ -421,7 +384,7 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, size_t ninfo, n; pmix_info_t *pinfo; opal_value_t *ival; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "%s PMIx_client get on proc %s key %s", @@ -434,7 +397,7 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == proc->jobid) { job = jptr; break; @@ -469,7 +432,7 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, n=0; OPAL_LIST_FOREACH(ival, info, opal_value_t) { (void)strncpy(pinfo[n].key, ival->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, ival); + pmix2x_value_load(&pinfo[n].value, ival); } } else { pinfo = NULL; @@ -486,11 +449,11 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, ret = OPAL_SUCCESS; } else { *val = OBJ_NEW(opal_value_t); - ret = pmix1_value_unload(*val, kv); + ret = pmix2x_value_unload(*val, kv); PMIX_VALUE_FREE(kv, 1); } } else { - ret = pmix1_convert_rc(rc); + ret = pmix2x_convert_rc(rc); } PMIX_INFO_FREE(pinfo, ninfo); return ret; @@ -499,13 +462,13 @@ int pmix1_get(const opal_process_name_t *proc, const char *key, static void val_cbfunc(pmix_status_t status, pmix_value_t *kv, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata; int rc; opal_value_t val, *v=NULL; - rc = pmix1_convert_opalrc(status); + rc = pmix2x_convert_opalrc(status); if (PMIX_SUCCESS == status && NULL != kv) { - rc = pmix1_value_unload(&val, kv); + rc = pmix2x_value_unload(&val, kv); v = &val; } @@ -515,15 +478,18 @@ static void val_cbfunc(pmix_status_t status, OBJ_RELEASE(op); } -int pmix1_getnb(const opal_process_name_t *proc, const char *key, +int pmix2x_getnb(const opal_process_name_t *proc, const char *key, opal_list_t *info, opal_pmix_value_cbfunc_t cbfunc, void *cbdata) { - pmix1_opcaddy_t *op; + pmix2x_opcaddy_t *op; pmix_status_t rc; size_t n; opal_value_t *ival; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access shared lists/objects */ opal_output_verbose(1, opal_pmix_base_framework.framework_output, "%s PMIx_client get_nb on proc %s key %s", @@ -531,7 +497,7 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, (NULL == proc) ? "NULL" : OPAL_NAME_PRINT(*proc), key); /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->valcbfunc = cbfunc; op->cbdata = cbdata; @@ -539,7 +505,7 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == proc->jobid) { job = jptr; break; @@ -562,7 +528,7 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, n=0; OPAL_LIST_FOREACH(ival, info, opal_value_t) { (void)strncpy(op->info[n].key, ival->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, ival); + pmix2x_value_load(&op->info[n].value, ival); } } } @@ -573,10 +539,10 @@ int pmix1_getnb(const opal_process_name_t *proc, const char *key, OBJ_RELEASE(op); } - return pmix1_convert_rc(rc); + return pmix2x_convert_rc(rc); } -int pmix1_publish(opal_list_t *info) +int pmix2x_publish(opal_list_t *info) { pmix_info_t *pinfo; pmix_status_t ret; @@ -596,7 +562,7 @@ int pmix1_publish(opal_list_t *info) n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, iptr); + pmix2x_value_load(&pinfo[n].value, iptr); ++n; } } else { @@ -605,16 +571,16 @@ int pmix1_publish(opal_list_t *info) ret = PMIx_Publish(pinfo, sz); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_publishnb(opal_list_t *info, +int pmix2x_publishnb(opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; opal_value_t *iptr; size_t n; - pmix1_opcaddy_t *op; + pmix2x_opcaddy_t *op; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client publish_nb"); @@ -624,7 +590,7 @@ int pmix1_publishnb(opal_list_t *info, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; @@ -634,17 +600,17 @@ int pmix1_publishnb(opal_list_t *info, n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, iptr); + pmix2x_value_load(&op->info[n].value, iptr); ++n; } } ret = PMIx_Publish_nb(op->info, op->sz, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_lookup(opal_list_t *data, opal_list_t *info) +int pmix2x_lookup(opal_list_t *data, opal_list_t *info) { pmix_pdata_t *pdata; pmix_info_t *pinfo; @@ -653,8 +619,10 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) pmix_status_t ret; opal_pmix_pdata_t *d; opal_value_t *iptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; + /* we must threadshift this request as we might not be in an event + * and we are going to access shared lists/objects */ opal_output_verbose(1, opal_pmix_base_framework.framework_output, "PMIx_client lookup"); @@ -675,7 +643,7 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(pinfo[n++].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, iptr); + pmix2x_value_load(&pinfo[n].value, iptr); ++n; } } else { @@ -690,7 +658,7 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) /* transfer the data back */ n=0; OPAL_LIST_FOREACH(d, data, opal_pmix_pdata_t) { - if (mca_pmix_pmix114_component.native_launch) { + if (mca_pmix_pmix2x_component.native_launch) { /* if we were launched by the OMPI RTE, then * the jobid is in a special format - so get it */ opal_convert_string_to_jobid(&d->proc.jobid, pdata[n].proc.nspace); @@ -701,24 +669,24 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) } /* if we don't already have it, add this to our jobid tracker */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == d->proc.jobid) { job = jptr; break; } } if (NULL == job) { - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); (void)strncpy(job->nspace, pdata[n].proc.nspace, PMIX_MAX_NSLEN); job->jobid = d->proc.jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); } if (PMIX_RANK_WILDCARD == pdata[n].proc.rank) { d->proc.vpid = OPAL_VPID_WILDCARD; } else { d->proc.vpid = pdata[n].proc.rank; } - rc = pmix1_value_unload(&d->value, &pdata[n].value); + rc = pmix2x_value_unload(&d->value, &pdata[n].value); if (OPAL_SUCCESS != rc) { OPAL_ERROR_LOG(rc); PMIX_PDATA_FREE(pdata, sz); @@ -728,32 +696,36 @@ int pmix1_lookup(opal_list_t *data, opal_list_t *info) } } - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } static void lk_cbfunc(pmix_status_t status, pmix_pdata_t data[], size_t ndata, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata; opal_pmix_pdata_t *d; opal_list_t results, *r = NULL; int rc; size_t n; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; - if (NULL == op->lkcbfunc) { + /* this is in the PMIx local thread - need to threadshift to + * our own thread as we will be accessing framework-global + * lists and objects */ + + if (NULL == op->lkcbfunc) { OBJ_RELEASE(op); return; } - rc = pmix1_convert_rc(status); + rc = pmix2x_convert_rc(status); if (OPAL_SUCCESS == rc) { OBJ_CONSTRUCT(&results, opal_list_t); for (n=0; n < ndata; n++) { d = OBJ_NEW(opal_pmix_pdata_t); opal_list_append(&results, &d->super); - if (mca_pmix_pmix114_component.native_launch) { + if (mca_pmix_pmix2x_component.native_launch) { /* if we were launched by the OMPI RTE, then * the jobid is in a special format - so get it */ opal_convert_string_to_jobid(&d->proc.jobid, data[n].proc.nspace); @@ -764,17 +736,17 @@ static void lk_cbfunc(pmix_status_t status, } /* if we don't already have it, add this to our jobid tracker */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == d->proc.jobid) { job = jptr; break; } } if (NULL == job) { - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); (void)strncpy(job->nspace, data[n].proc.nspace, PMIX_MAX_NSLEN); job->jobid = d->proc.jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); } if (PMIX_RANK_WILDCARD == data[n].proc.rank) { d->proc.vpid = OPAL_VPID_WILDCARD; @@ -782,7 +754,7 @@ static void lk_cbfunc(pmix_status_t status, d->proc.vpid = data[n].proc.rank; } d->value.key = strdup(data[n].key); - rc = pmix1_value_unload(&d->value, &data[n].value); + rc = pmix2x_value_unload(&d->value, &data[n].value); if (OPAL_SUCCESS != rc) { rc = OPAL_ERR_BAD_PARAM; OPAL_ERROR_LOG(rc); @@ -801,11 +773,11 @@ static void lk_cbfunc(pmix_status_t status, OBJ_RELEASE(op); } -int pmix1_lookupnb(char **keys, opal_list_t *info, +int pmix2x_lookupnb(char **keys, opal_list_t *info, opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; - pmix1_opcaddy_t *op; + pmix2x_opcaddy_t *op; opal_value_t *iptr; size_t n; @@ -814,7 +786,7 @@ int pmix1_lookupnb(char **keys, opal_list_t *info, "PMIx_client lookup_nb"); /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->lkcbfunc = cbfunc; op->cbdata = cbdata; @@ -825,7 +797,7 @@ int pmix1_lookupnb(char **keys, opal_list_t *info, n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, iptr); + pmix2x_value_load(&op->info[n].value, iptr); ++n; } } @@ -833,10 +805,10 @@ int pmix1_lookupnb(char **keys, opal_list_t *info, ret = PMIx_Lookup_nb(keys, op->info, op->sz, lk_cbfunc, op); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_unpublish(char **keys, opal_list_t *info) +int pmix2x_unpublish(char **keys, opal_list_t *info) { pmix_status_t ret; size_t ninfo, n; @@ -849,7 +821,7 @@ int pmix1_unpublish(char **keys, opal_list_t *info) n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(pinfo[n++].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, iptr); + pmix2x_value_load(&pinfo[n].value, iptr); ++n; } } else { @@ -860,19 +832,19 @@ int pmix1_unpublish(char **keys, opal_list_t *info) ret = PMIx_Unpublish(keys, pinfo, ninfo); PMIX_INFO_FREE(pinfo, ninfo); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_unpublishnb(char **keys, opal_list_t *info, +int pmix2x_unpublishnb(char **keys, opal_list_t *info, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; - pmix1_opcaddy_t *op; + pmix2x_opcaddy_t *op; opal_value_t *iptr; size_t n; /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; @@ -883,7 +855,7 @@ int pmix1_unpublishnb(char **keys, opal_list_t *info, n=0; OPAL_LIST_FOREACH(iptr, info, opal_value_t) { (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, iptr); + pmix2x_value_load(&op->info[n].value, iptr); ++n; } } @@ -891,10 +863,10 @@ int pmix1_unpublishnb(char **keys, opal_list_t *info, ret = PMIx_Unpublish_nb(keys, op->info, op->sz, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) +int pmix2x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) { pmix_status_t ret; pmix_info_t *pinfo = NULL; @@ -903,14 +875,14 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) char nspace[PMIX_MAX_NSLEN+1]; opal_value_t *info; opal_pmix_app_t *app; - opal_pmix1_jobid_trkr_t *job; + opal_pmix2x_jobid_trkr_t *job; if (NULL != job_info && 0 < (ninfo = opal_list_get_size(job_info))) { PMIX_INFO_CREATE(pinfo, ninfo); n=0; OPAL_LIST_FOREACH(info, job_info, opal_value_t) { (void)strncpy(pinfo[n].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&pinfo[n].value, info); + pmix2x_value_load(&pinfo[n].value, info); ++n; } } @@ -929,7 +901,7 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) m=0; OPAL_LIST_FOREACH(info, &app->info, opal_value_t) { (void)strncpy(papps[n].info[m].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&papps[n].info[m].value, info); + pmix2x_value_load(&papps[n].info[m].value, info); ++m; } } @@ -938,7 +910,7 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) ret = PMIx_Spawn(pinfo, ninfo, papps, napps, nspace); if (PMIX_SUCCESS == ret) { - if (mca_pmix_pmix114_component.native_launch) { + if (mca_pmix_pmix2x_component.native_launch) { /* if we were launched by the OMPI RTE, then * the jobid is in a special format - so get it */ opal_convert_string_to_jobid(jobid, nspace); @@ -948,27 +920,31 @@ int pmix1_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) OPAL_HASH_STR(nspace, *jobid); } /* add this to our jobid tracker */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); job->jobid = *jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); } PMIX_APP_FREE(papps, napps); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } static void spcbfunc(pmix_status_t status, char *nspace, void *cbdata) { - pmix1_opcaddy_t *op = (pmix1_opcaddy_t*)cbdata; + pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata; int rc; opal_jobid_t jobid=OPAL_JOBID_INVALID; - opal_pmix1_jobid_trkr_t *job; + opal_pmix2x_jobid_trkr_t *job; - rc = pmix1_convert_rc(status); + /* this is in the PMIx local thread - need to threadshift to + * our own thread as we will be accessing framework-global + * lists and objects */ + + rc = pmix2x_convert_rc(status); if (PMIX_SUCCESS == status) { - if (mca_pmix_pmix114_component.native_launch) { + if (mca_pmix_pmix2x_component.native_launch) { /* if we were launched by the OMPI RTE, then * the jobid is in a special format - so get it */ opal_convert_string_to_jobid(&jobid, nspace); @@ -978,27 +954,27 @@ static void spcbfunc(pmix_status_t status, OPAL_HASH_STR(nspace, jobid); } /* add this to our jobid tracker */ - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); job->jobid = jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); } op->spcbfunc(rc, jobid, op->cbdata); OBJ_RELEASE(op); } -int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, +int pmix2x_spawnnb(opal_list_t *job_info, opal_list_t *apps, opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; - pmix1_opcaddy_t *op; + pmix2x_opcaddy_t *op; size_t n, m; opal_value_t *info; opal_pmix_app_t *app; /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->spcbfunc = cbfunc; op->cbdata = cbdata; @@ -1007,7 +983,7 @@ int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, n=0; OPAL_LIST_FOREACH(info, job_info, opal_value_t) { (void)strncpy(op->info[n].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->info[n].value, info); + pmix2x_value_load(&op->info[n].value, info); ++n; } } @@ -1026,7 +1002,7 @@ int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, m=0; OPAL_LIST_FOREACH(info, &app->info, opal_value_t) { (void)strncpy(op->apps[n].info[m].key, info->key, PMIX_MAX_KEYLEN); - pmix1_value_load(&op->apps[n].info[m].value, info); + pmix2x_value_load(&op->apps[n].info[m].value, info); ++m; } } @@ -1035,16 +1011,16 @@ int pmix1_spawnnb(opal_list_t *job_info, opal_list_t *apps, ret = PMIx_Spawn_nb(op->info, op->ninfo, op->apps, op->sz, spcbfunc, op); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_connect(opal_list_t *procs) +int pmix2x_connect(opal_list_t *procs) { pmix_status_t ret; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1059,14 +1035,13 @@ int pmix1_connect(opal_list_t *procs) /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == ptr->name.jobid) { job = jptr; break; } } if (NULL == job) { - free(parray); OPAL_ERROR_LOG(OPAL_ERR_NOT_FOUND); return OPAL_ERR_NOT_FOUND; } @@ -1082,18 +1057,21 @@ int pmix1_connect(opal_list_t *procs) ret = PMIx_Connect(parray, cnt, NULL, 0); PMIX_PROC_FREE(parray, cnt); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_connectnb(opal_list_t *procs, +int pmix2x_connectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; size_t n, cnt=0; opal_namelist_t *ptr; - pmix1_opcaddy_t *op; - opal_pmix1_jobid_trkr_t *job; + pmix2x_opcaddy_t *op; + opal_pmix2x_jobid_trkr_t *job; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1101,7 +1079,7 @@ int pmix1_connectnb(opal_list_t *procs, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; op->nprocs = cnt; @@ -1113,7 +1091,7 @@ int pmix1_connectnb(opal_list_t *procs, OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { /* look thru our list of jobids and find the * corresponding nspace */ - OPAL_LIST_FOREACH(job, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(job, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (job->jobid == ptr->name.jobid) { (void)strncpy(op->procs[n].nspace, job->nspace, PMIX_MAX_NSLEN); break; @@ -1129,16 +1107,16 @@ int pmix1_connectnb(opal_list_t *procs, ret = PMIx_Connect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_disconnect(opal_list_t *procs) +int pmix2x_disconnect(opal_list_t *procs) { pmix_status_t ret; pmix_proc_t *parray=NULL; size_t n, cnt=0; opal_namelist_t *ptr; - opal_pmix1_jobid_trkr_t *job; + opal_pmix2x_jobid_trkr_t *job; /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1152,7 +1130,7 @@ int pmix1_disconnect(opal_list_t *procs) OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { /* look thru our list of jobids and find the * corresponding nspace */ - OPAL_LIST_FOREACH(job, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(job, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (job->jobid == ptr->name.jobid) { (void)strncpy(parray[n].nspace, job->nspace, PMIX_MAX_NSLEN); break; @@ -1169,18 +1147,21 @@ int pmix1_disconnect(opal_list_t *procs) ret = PMIx_Disconnect(parray, cnt, NULL, 0); PMIX_PROC_FREE(parray, cnt); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_disconnectnb(opal_list_t *procs, +int pmix2x_disconnectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_status_t ret; size_t n, cnt=0; opal_namelist_t *ptr; - pmix1_opcaddy_t *op; - opal_pmix1_jobid_trkr_t *job; + pmix2x_opcaddy_t *op; + opal_pmix2x_jobid_trkr_t *job; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ /* protect against bozo error */ if (NULL == procs || 0 == (cnt = opal_list_get_size(procs))) { @@ -1188,7 +1169,7 @@ int pmix1_disconnectnb(opal_list_t *procs, } /* create the caddy */ - op = OBJ_NEW(pmix1_opcaddy_t); + op = OBJ_NEW(pmix2x_opcaddy_t); op->opcbfunc = cbfunc; op->cbdata = cbdata; op->nprocs = cnt; @@ -1200,7 +1181,7 @@ int pmix1_disconnectnb(opal_list_t *procs, OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { /* look thru our list of jobids and find the * corresponding nspace */ - OPAL_LIST_FOREACH(job, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(job, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (job->jobid == ptr->name.jobid) { (void)strncpy(op->procs[n].nspace, job->nspace, PMIX_MAX_NSLEN); break; @@ -1216,11 +1197,11 @@ int pmix1_disconnectnb(opal_list_t *procs, ret = PMIx_Disconnect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); - return pmix1_convert_rc(ret); + return pmix2x_convert_rc(ret); } -int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, +int pmix2x_resolve_peers(const char *nodename, opal_jobid_t jobid, opal_list_t *procs) { char *nspace; @@ -1229,13 +1210,16 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, opal_namelist_t *nm; int rc; pmix_status_t ret; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ if (OPAL_JOBID_WILDCARD == jobid) { nspace = NULL; } else { job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == jobid) { job = jptr; break; @@ -1248,13 +1232,13 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, } ret = PMIx_Resolve_peers(nodename, nspace, &array, &nprocs); - rc = pmix1_convert_rc(ret); + rc = pmix2x_convert_rc(ret); if (NULL != array && 0 < nprocs) { for (n=0; n < nprocs; n++) { nm = OBJ_NEW(opal_namelist_t); opal_list_append(procs, &nm->super); - if (mca_pmix_pmix114_component.native_launch) { + if (mca_pmix_pmix2x_component.native_launch) { /* if we were launched by the OMPI RTE, then * the jobid is in a special format - so get it */ opal_convert_string_to_jobid(&nm->name.jobid, array[n].nspace); @@ -1265,17 +1249,17 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, } /* if we don't already have it, add this to our jobid tracker */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == nm->name.jobid) { job = jptr; break; } } if (NULL == job) { - job = OBJ_NEW(opal_pmix1_jobid_trkr_t); + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); job->jobid = jobid; - opal_list_append(&mca_pmix_pmix114_component.jobids, &job->super); + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); } nm->name.vpid = array[n].rank; } @@ -1285,17 +1269,20 @@ int pmix1_resolve_peers(const char *nodename, opal_jobid_t jobid, return rc; } -int pmix1_resolve_nodes(opal_jobid_t jobid, char **nodelist) +int pmix2x_resolve_nodes(opal_jobid_t jobid, char **nodelist) { pmix_status_t ret; char *nspace=NULL; - opal_pmix1_jobid_trkr_t *job, *jptr; + opal_pmix2x_jobid_trkr_t *job, *jptr; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ if (OPAL_JOBID_WILDCARD != jobid) { /* look thru our list of jobids and find the * corresponding nspace */ job = NULL; - OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix114_component.jobids, opal_pmix1_jobid_trkr_t) { + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { if (jptr->jobid == jobid) { job = jptr; break; @@ -1309,5 +1296,5 @@ int pmix1_resolve_nodes(opal_jobid_t jobid, char **nodelist) ret = PMIx_Resolve_nodes(nspace, nodelist); - return pmix1_convert_rc(ret);; + return pmix2x_convert_rc(ret);; } diff --git a/opal/mca/pmix/pmix2x/pmix2x_component.c b/opal/mca/pmix/pmix2x/pmix2x_component.c new file mode 100644 index 0000000000..e7d51f1a22 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix2x_component.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "opal_config.h" + +#include "opal/constants.h" +#include "opal/class/opal_list.h" +#include "opal/util/proc.h" +#include "opal/mca/pmix/pmix.h" +#include "pmix2x.h" + +/* + * Public string showing the pmix external component version number + */ +const char *opal_pmix_pmix2x_component_version_string = + "OPAL pmix2x MCA component version " OPAL_VERSION; + +/* + * Local function + */ +static int external_open(void); +static int external_close(void); +static int external_component_query(mca_base_module_t **module, int *priority); +static int external_register(void); + + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ + +mca_pmix_pmix2x_component_t mca_pmix_pmix2x_component = { + { + /* First, the mca_component_t struct containing meta information + about the component itself */ + + .base_version = { + /* Indicate that we are a pmix v1.1.0 component (which also + implies a specific MCA version) */ + + OPAL_PMIX_BASE_VERSION_2_0_0, + + /* Component name and version */ + + .mca_component_name = "pmix2x", + MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, + OPAL_RELEASE_VERSION), + + /* Component open and close functions */ + + .mca_open_component = external_open, + .mca_close_component = external_close, + .mca_query_component = external_component_query, + .mca_register_component_params = external_register, + }, + /* Next the MCA v1.0.0 component meta data */ + .base_data = { + /* The component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + } + }, + .native_launch = false +}; + +static int external_register(void) +{ + mca_pmix_pmix2x_component.cache_size = 256; + mca_base_component_var_register(&mca_pmix_pmix2x_component.super.base_version, + "cache_size", "Size of the ring buffer cache for events", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, + MCA_BASE_VAR_SCOPE_CONSTANT, + &mca_pmix_pmix2x_component.cache_size); + + return OPAL_SUCCESS; +} + + +static int external_open(void) +{ + mca_pmix_pmix2x_component.evindex = 0; + OBJ_CONSTRUCT(&mca_pmix_pmix2x_component.jobids, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_pmix2x_component.single_events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_pmix2x_component.multi_events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_pmix2x_component.default_events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_pmix2x_component.cache, opal_list_t); + + return OPAL_SUCCESS; +} + +static int external_close(void) +{ + OPAL_LIST_DESTRUCT(&mca_pmix_pmix2x_component.jobids); + OPAL_LIST_DESTRUCT(&mca_pmix_pmix2x_component.single_events); + OPAL_LIST_DESTRUCT(&mca_pmix_pmix2x_component.multi_events); + OPAL_LIST_DESTRUCT(&mca_pmix_pmix2x_component.default_events); + OPAL_LIST_DESTRUCT(&mca_pmix_pmix2x_component.cache); + return OPAL_SUCCESS; +} + + +static int external_component_query(mca_base_module_t **module, int *priority) +{ + char *t, *id; + + /* see if a PMIx server is present */ + if (NULL != (t = getenv("PMIX_SERVER_URI")) || + NULL != (id = getenv("PMIX_ID"))) { + /* if PMIx is present, then we are a client and need to use it */ + *priority = 100; + } else { + /* we could be a server, so we still need to be considered */ + *priority = 5; + } + *module = (mca_base_module_t *)&opal_pmix_pmix2x_module; + return OPAL_SUCCESS; +} diff --git a/opal/mca/pmix/pmix114/pmix1_server_north.c b/opal/mca/pmix/pmix2x/pmix2x_server_north.c similarity index 63% rename from opal/mca/pmix/pmix114/pmix1_server_north.c rename to opal/mca/pmix/pmix2x/pmix2x_server_north.c index 058a273a20..898dda9b6d 100644 --- a/opal/mca/pmix/pmix114/pmix1_server_north.c +++ b/opal/mca/pmix/pmix2x/pmix2x_server_north.c @@ -1,9 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2014 Mellanox Technologies, Inc. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -34,7 +34,7 @@ #include "opal/util/proc.h" #include "opal/util/show_help.h" #include "opal/mca/pmix/base/base.h" -#include "pmix1.h" +#include "pmix2x.h" #include "pmix.h" #include "pmix_server.h" @@ -44,124 +44,135 @@ /* These are the interfaces used by the embedded PMIx server * to call up into ORTE for service requests */ -static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object); -static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object, - int status, const char msg[], - pmix_proc_t procs[], size_t nprocs, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_publish_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_lookup_fn(const pmix_proc_t *proc, char **keys, const pmix_info_t info[], size_t ninfo, - char *data, size_t ndata, - pmix_modex_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *proc, - const pmix_info_t info[], size_t ninfo, - pmix_modex_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_publish_fn(const pmix_proc_t *proc, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_lookup_fn(const pmix_proc_t *proc, char **keys, - const pmix_info_t info[], size_t ninfo, - pmix_lookup_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_unpublish_fn(const pmix_proc_t *proc, char **keys, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_spawn_fn(const pmix_proc_t *proc, - const pmix_info_t job_info[], size_t ninfo, - const pmix_app_t apps[], size_t napps, - pmix_spawn_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + pmix_lookup_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_unpublish_fn(const pmix_proc_t *proc, char **keys, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_register_events(const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_deregister_events(const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_listener_fn(int listening_sd, - pmix_connection_cbfunc_t cbfunc); - -pmix_server_module_t pmix114_module = { - server_client_connected_fn, - server_client_finalized_fn, - server_abort_fn, - server_fencenb_fn, - server_dmodex_req_fn, - server_publish_fn, - server_lookup_fn, - server_unpublish_fn, - server_spawn_fn, - server_connect_fn, - server_disconnect_fn, - server_register_events, - server_deregister_events, - server_listener_fn + static pmix_status_t server_spawn_fn(const pmix_proc_t *proc, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); + static pmix_status_t server_notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + pmix_server_module_t mymodule = { + .client_connected = server_client_connected_fn, + .client_finalized = server_client_finalized_fn, + .abort = server_abort_fn, + .fence_nb = server_fencenb_fn, + .direct_modex = server_dmodex_req_fn, + .publish = server_publish_fn, + .lookup = server_lookup_fn, + .unpublish = server_unpublish_fn, + .spawn = server_spawn_fn, + .connect = server_connect_fn, + .disconnect = server_disconnect_fn, + .register_events = server_register_events, + .deregister_events = server_deregister_events, + .notify_event = server_notify_event }; -opal_pmix_server_module_t *pmix114_host_module = NULL; +opal_pmix_server_module_t *host_module = NULL; static void opal_opcbfunc(int status, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix2x_opalcaddy_t *opalcaddy = (pmix2x_opalcaddy_t*)cbdata; if (NULL != opalcaddy->opcbfunc) { - opalcaddy->opcbfunc(pmix1_convert_opalrc(status), opalcaddy->cbdata); + opalcaddy->opcbfunc(pmix2x_convert_opalrc(status), opalcaddy->cbdata); } OBJ_RELEASE(opalcaddy); } -static pmix_status_t server_client_connected_fn(const pmix_proc_t *p, void *server_object) +static pmix_status_t server_client_connected_fn(const pmix_proc_t *p, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; opal_process_name_t proc; + pmix2x_opalcaddy_t *opalcaddy; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->client_connected) { + if (NULL == host_module || NULL == host_module->client_connected) { return PMIX_SUCCESS; } + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } proc.vpid = p->rank; /* pass it up */ - rc = pmix114_host_module->client_connected(&proc, server_object); - return pmix1_convert_opalrc(rc); + rc = host_module->client_connected(&proc, server_object, + opal_opcbfunc, opalcaddy); + return pmix2x_convert_opalrc(rc); } static pmix_status_t server_client_finalized_fn(const pmix_proc_t *p, void* server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_process_name_t proc; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->client_finalized) { + if (NULL == host_module || NULL == host_module->client_finalized) { return PMIX_SUCCESS; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } proc.vpid = p->rank; /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; /* pass it up */ - rc = pmix114_host_module->client_finalized(&proc, server_object, opal_opcbfunc, opalcaddy); + rc = host_module->client_finalized(&proc, server_object, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, @@ -173,20 +184,20 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, opal_namelist_t *nm; opal_process_name_t proc; int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->abort) { + if (NULL == host_module || NULL == host_module->abort) { return PMIX_ERR_NOT_SUPPORTED; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } proc.vpid = p->rank; /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -196,7 +207,7 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -206,17 +217,17 @@ static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, } /* pass it up */ - rc = pmix114_host_module->abort(&proc, server_object, status, msg, + rc = host_module->abort(&proc, server_object, status, msg, &opalcaddy->procs, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } static void _data_release(void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix2x_opalcaddy_t *opalcaddy = (pmix2x_opalcaddy_t*)cbdata; if (NULL != opalcaddy->odmdxfunc) { opalcaddy->odmdxfunc(opalcaddy->ocbdata); @@ -228,9 +239,9 @@ static void opmdx_response(int status, const char *data, size_t sz, void *cbdata opal_pmix_release_cbfunc_t relcbfunc, void *relcbdata) { pmix_status_t rc; - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix2x_opalcaddy_t *opalcaddy = (pmix2x_opalcaddy_t*)cbdata; - rc = pmix1_convert_rc(status); + rc = pmix2x_convert_rc(status); if (NULL != opalcaddy->mdxcbfunc) { opalcaddy->odmdxfunc = relcbfunc; opalcaddy->ocbdata = relcbdata; @@ -246,18 +257,18 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; size_t n; opal_namelist_t *nm; opal_value_t *iptr; int rc; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->fence_nb) { + if (NULL == host_module || NULL == host_module->fence_nb) { return PMIX_ERR_NOT_SUPPORTED; } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->mdxcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -267,7 +278,7 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -281,19 +292,19 @@ static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->fence_nb(&opalcaddy->procs, &opalcaddy->info, + rc = host_module->fence_nb(&opalcaddy->procs, &opalcaddy->info, data, ndata, opmdx_response, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, @@ -301,18 +312,18 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, pmix_modex_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *iptr; size_t n; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->direct_modex) { + if (NULL == host_module || NULL == host_module->direct_modex) { return PMIX_ERR_NOT_SUPPORTED; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -321,7 +332,7 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->mdxcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -330,21 +341,21 @@ static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->direct_modex(&proc, &opalcaddy->info, opmdx_response, opalcaddy); + rc = host_module->direct_modex(&proc, &opalcaddy->info, opmdx_response, opalcaddy); if (OPAL_SUCCESS != rc && OPAL_ERR_IN_PROCESS != rc) { OBJ_RELEASE(opalcaddy); } if (OPAL_ERR_IN_PROCESS == rc) { rc = OPAL_SUCCESS; } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } static pmix_status_t server_publish_fn(const pmix_proc_t *p, @@ -353,17 +364,17 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, { int rc; size_t n; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *oinfo; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->publish) { + if (NULL == host_module || NULL == host_module->publish) { return PMIX_ERR_NOT_SUPPORTED; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -372,7 +383,7 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -381,33 +392,33 @@ static pmix_status_t server_publish_fn(const pmix_proc_t *p, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->publish(&proc, &opalcaddy->info, opal_opcbfunc, opalcaddy); + rc = host_module->publish(&proc, &opalcaddy->info, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } static void opal_lkupcbfunc(int status, opal_list_t *data, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix2x_opalcaddy_t *opalcaddy = (pmix2x_opalcaddy_t*)cbdata; pmix_status_t rc; pmix_pdata_t *d=NULL; size_t nd=0, n; opal_pmix_pdata_t *p; if (NULL != opalcaddy->lkupcbfunc) { - rc = pmix1_convert_opalrc(status); + rc = pmix2x_convert_opalrc(status); /* convert any returned data */ if (NULL != data) { nd = opal_list_get_size(data); @@ -418,7 +429,7 @@ static void opal_lkupcbfunc(int status, (void)opal_snprintf_jobid(d[n].proc.nspace, PMIX_MAX_NSLEN, p->proc.jobid); d[n].proc.rank = p->proc.vpid; (void)strncpy(d[n].key, p->value.key, PMIX_MAX_KEYLEN); - pmix1_value_load(&d[n].value, &p->value); + pmix2x_value_load(&d[n].value, &p->value); } } opalcaddy->lkupcbfunc(rc, d, nd, opalcaddy->cbdata); @@ -431,18 +442,18 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, pmix_lookup_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *iptr; size_t n; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->lookup) { + if (NULL == host_module || NULL == host_module->lookup) { return PMIX_ERR_NOT_SUPPORTED; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -451,7 +462,7 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->lkupcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -460,19 +471,19 @@ static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->lookup(&proc, keys, &opalcaddy->info, opal_lkupcbfunc, opalcaddy); + rc = host_module->lookup(&proc, keys, &opalcaddy->info, opal_lkupcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } @@ -481,18 +492,18 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_value_t *iptr; size_t n; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->unpublish) { + if (NULL == host_module || NULL == host_module->unpublish) { return PMIX_SUCCESS; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -501,7 +512,7 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -510,29 +521,29 @@ static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, iptr = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &iptr->super); iptr->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(iptr, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(iptr, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->unpublish(&proc, keys, &opalcaddy->info, opal_opcbfunc, opalcaddy); + rc = host_module->unpublish(&proc, keys, &opalcaddy->info, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } static void opal_spncbfunc(int status, opal_jobid_t jobid, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy = (pmix1_opalcaddy_t*)cbdata; + pmix2x_opalcaddy_t *opalcaddy = (pmix2x_opalcaddy_t*)cbdata; pmix_status_t rc; char nspace[PMIX_MAX_NSLEN]; if (NULL != opalcaddy->spwncbfunc) { - rc = pmix1_convert_opalrc(status); + rc = pmix2x_convert_opalrc(status); /* convert the jobid */ (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid); opalcaddy->spwncbfunc(rc, nspace, opalcaddy->cbdata); @@ -545,20 +556,20 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, const pmix_app_t apps[], size_t napps, pmix_spawn_cbfunc_t cbfunc, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_process_name_t proc; opal_pmix_app_t *app; opal_value_t *oinfo; size_t k, n; int rc; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->spawn) { + if (NULL == host_module || NULL == host_module->spawn) { return PMIX_ERR_NOT_SUPPORTED; } /* convert the nspace/rank to an opal_process_name_t */ if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == p->rank) { proc.vpid = OPAL_VPID_WILDCARD; @@ -567,7 +578,7 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->spwncbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -576,9 +587,9 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(job_info[k].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &job_info[k].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &job_info[k].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } @@ -601,21 +612,21 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&app->info, &oinfo->super); oinfo->key = strdup(apps[n].info[k].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &apps[n].info[k].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &apps[n].info[k].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } } /* pass it up */ - rc = pmix114_host_module->spawn(&proc, &opalcaddy->info, &opalcaddy->apps, opal_spncbfunc, opalcaddy); + rc = host_module->spawn(&proc, &opalcaddy->info, &opalcaddy->apps, opal_spncbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OPAL_ERROR_LOG(rc); OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } @@ -624,17 +635,17 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_namelist_t *nm; size_t n; opal_value_t *oinfo; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->connect) { + if (NULL == host_module || NULL == host_module->connect) { return PMIX_ERR_NOT_SUPPORTED; } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -644,7 +655,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -658,19 +669,19 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->connect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); + rc = host_module->connect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } @@ -679,17 +690,17 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; opal_namelist_t *nm; size_t n; opal_value_t *oinfo; - if (NULL == pmix114_host_module || NULL == pmix114_host_module->disconnect) { + if (NULL == host_module || NULL == host_module->disconnect) { return PMIX_ERR_NOT_SUPPORTED; } /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -699,7 +710,7 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro opal_list_append(&opalcaddy->procs, &nm->super); if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } if (PMIX_RANK_WILDCARD == procs[n].rank) { nm->name.vpid = OPAL_VPID_WILDCARD; @@ -713,31 +724,32 @@ static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t npro oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->disconnect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); + rc = host_module->disconnect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } -static pmix_status_t server_register_events(const pmix_info_t info[], size_t ninfo, +static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { - pmix1_opalcaddy_t *opalcaddy; + pmix2x_opalcaddy_t *opalcaddy; size_t n; opal_value_t *oinfo; int rc; /* setup the caddy */ - opalcaddy = OBJ_NEW(pmix1_opalcaddy_t); + opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t); opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; @@ -746,35 +758,32 @@ static pmix_status_t server_register_events(const pmix_info_t info[], size_t nin oinfo = OBJ_NEW(opal_value_t); opal_list_append(&opalcaddy->info, &oinfo->super); oinfo->key = strdup(info[n].key); - if (OPAL_SUCCESS != (rc = pmix1_value_unload(oinfo, &info[n].value))) { + if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &info[n].value))) { OBJ_RELEASE(opalcaddy); - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } } /* pass it up */ - rc = pmix114_host_module->register_events(&opalcaddy->info, opal_opcbfunc, opalcaddy); + rc = host_module->register_events(&opalcaddy->info, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } - return pmix1_convert_opalrc(rc); + return pmix2x_convert_opalrc(rc); } -static pmix_status_t server_deregister_events(const pmix_info_t info[], size_t ninfo, +static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes, pmix_op_cbfunc_t cbfunc, void *cbdata) { - return PMIX_ERR_NOT_IMPLEMENTED; + return PMIX_ERR_NOT_SUPPORTED; } -static pmix_status_t server_listener_fn(int listening_sd, - pmix_connection_cbfunc_t cbfunc) + +static pmix_status_t server_notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) { - int rc; - - if (NULL == pmix114_host_module || NULL == pmix114_host_module->listener) { - return PMIX_ERR_NOT_SUPPORTED; - } - - rc = pmix114_host_module->listener(listening_sd, cbfunc); - return pmix1_convert_opalrc(rc); + return PMIX_ERR_NOT_SUPPORTED; } diff --git a/opal/mca/pmix/pmix2x/pmix2x_server_south.c b/opal/mca/pmix/pmix2x/pmix2x_server_south.c new file mode 100644 index 0000000000..79c7fbbd54 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix2x_server_south.c @@ -0,0 +1,508 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" +#include "opal/types.h" + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "opal/dss/dss.h" +#include "opal/mca/event/event.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_progress_threads.h" +#include "opal/util/argv.h" +#include "opal/util/error.h" +#include "opal/util/output.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" +#include "opal/mca/pmix/base/base.h" +#include "pmix2x.h" + +#include "pmix.h" +#include "pmix_server.h" + +/**** S.O.U.T.H.B.O.U.N.D I.N.T.E.R.F.A.C.E.S ****/ + +/* These are the interfaces used by the OMPI/ORTE/OPAL layer to call + * down into the embedded PMIx server. */ + +extern pmix_server_module_t mymodule; +extern opal_pmix_server_module_t *host_module; +static char *dbgvalue=NULL; +static size_t errhdler_ref = 0; + +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + } while (0) + +static void errreg_cbfunc (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + volatile bool *active = (volatile bool*)cbdata; + + errhdler_ref = errhandler_ref; + opal_output_verbose(5, opal_pmix_base_framework.framework_output, + "PMIX server errreg_cbfunc - error handler registered status=%d, reference=%lu", + status, (unsigned long)errhandler_ref); + *active = false; +} + +int pmix2x_server_init(opal_pmix_server_module_t *module, + opal_list_t *info) +{ + pmix_status_t rc; + int dbg; + opal_value_t *kv; + pmix_info_t *pinfo; + size_t sz, n; + volatile bool active; + + if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) { + asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg); + putenv(dbgvalue); + } + + /* convert the list to an array of pmix_info_t */ + if (NULL != info) { + sz = opal_list_get_size(info); + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix2x_value_load(&pinfo[n].value, kv); + ++n; + } + } else { + sz = 0; + pinfo = NULL; + } + + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, pinfo, sz))) { + PMIX_INFO_FREE(pinfo, sz); + return pmix2x_convert_rc(rc); + } + PMIX_INFO_FREE(pinfo, sz); + + /* record the host module */ + host_module = module; + + /* register the default event handler */ + active = true; + PMIx_Register_event_handler(NULL, 0, NULL, 0, pmix2x_event_hdlr, errreg_cbfunc, (void*)&active); + PMIX_WAIT_FOR_COMPLETION(active); + + return OPAL_SUCCESS; +} + +static void fincb(pmix_status_t status, void *cbdata) +{ + volatile bool *active = (volatile bool*)cbdata; + *active = false; +} + +int pmix2x_server_finalize(void) +{ + pmix_status_t rc; + volatile bool active; + + /* deregister the default event handler */ + active = true; + PMIx_Deregister_event_handler(errhdler_ref, fincb, (void*)&active); + PMIX_WAIT_FOR_COMPLETION(active); + + rc = PMIx_server_finalize(); + return pmix2x_convert_rc(rc); +} + +int pmix2x_server_gen_regex(const char *input, char **regex) +{ + pmix_status_t rc; + + rc = PMIx_generate_regex(input, regex); + return pmix2x_convert_rc(rc); +} + + +int pmix2x_server_gen_ppn(const char *input, char **ppn) +{ + pmix_status_t rc; + + rc = PMIx_generate_ppn(input, ppn); + return pmix2x_convert_rc(rc); +} + +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata; + + if (NULL != op->opcbfunc) { + op->opcbfunc(pmix2x_convert_rc(status), op->cbdata); + } + if (op->active) { + op->status = status; + op->active = false; + } else { + OBJ_RELEASE(op); + } +} + +static void _reg_nspace(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + opal_value_t *kv, *k2; + pmix_info_t *pinfo = NULL, *pmap; + size_t sz, szmap, m, n; + char nspace[PMIX_MAX_NSLEN]; + pmix_status_t rc; + opal_list_t *pmapinfo; + opal_pmix2x_jobid_trkr_t *job; + pmix2x_opcaddy_t op; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + + /* convert the jobid */ + (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, cd->jobid); + + /* store this job in our list of known nspaces */ + job = OBJ_NEW(opal_pmix2x_jobid_trkr_t); + (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); + job->jobid = cd->jobid; + opal_list_append(&mca_pmix_pmix2x_component.jobids, &job->super); + + /* convert the list to an array of pmix_info_t */ + if (NULL != cd->info) { + sz = opal_list_get_size(cd->info); + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, cd->info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + if (0 == strcmp(kv->key, OPAL_PMIX_PROC_DATA)) { + pinfo[n].value.type = PMIX_INFO_ARRAY; + /* the value contains a list of values - convert + * that list to another array */ + pmapinfo = (opal_list_t*)kv->data.ptr; + szmap = opal_list_get_size(pmapinfo); + PMIX_INFO_CREATE(pmap, szmap); + pinfo[n].value.data.array.array = (struct pmix_info_t*)pmap; + pinfo[n].value.data.array.size = szmap; + m = 0; + OPAL_LIST_FOREACH(k2, pmapinfo, opal_value_t) { + (void)strncpy(pmap[m].key, k2->key, PMIX_MAX_KEYLEN); + pmix2x_value_load(&pmap[m].value, k2); + ++m; + } + } else { + pmix2x_value_load(&pinfo[n].value, kv); + } + ++n; + } + } else { + sz = 0; + pinfo = NULL; + } + + OBJ_CONSTRUCT(&op, pmix2x_opcaddy_t); + op.active = true; + rc = PMIx_server_register_nspace(nspace, cd->status, pinfo, sz, + opcbfunc, (void*)&op); + if (PMIX_SUCCESS == rc) { + PMIX_WAIT_FOR_COMPLETION(op.active); + } else { + op.status = rc; + } + /* ensure we execute the cbfunc so the caller doesn't hang */ + if (NULL != cd->opcbfunc) { + cd->opcbfunc(pmix2x_convert_rc(op.status), cd->cbdata); + } + if (NULL != pinfo) { + PMIX_INFO_FREE(pinfo, sz); + } + OBJ_DESTRUCT(&op); + OBJ_RELEASE(cd); +} + +int pmix2x_server_register_nspace(opal_jobid_t jobid, + int nlocalprocs, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix2x_threadshift_t *cd; + + /* we must threadshift this request as it touches + * shared lists of objects */ + cd = OBJ_NEW(pmix2x_threadshift_t); + cd->jobid = jobid; + cd->status = nlocalprocs; + cd->info = info; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + /* if the cbfunc is NULL, then the caller is in an event + * and we can directly call the processing function */ + if (NULL == cbfunc) { + _reg_nspace(0, 0, cd); + } else { + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _reg_nspace, cd); + event_active(&cd->ev, EV_WRITE, 1); + } + + return OPAL_SUCCESS; +} + +static void tdcbfunc(pmix_status_t status, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + + if (NULL != cd->opcbfunc) { + cd->opcbfunc(pmix2x_convert_rc(status), cd->cbdata); + } + if (cd->active) { + cd->active = false; + } else { + OBJ_RELEASE(cd); + } +} + +static void _dereg_nspace(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + opal_pmix2x_jobid_trkr_t *jptr; + + /* if we don't already have it, we can ignore this */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { + if (jptr->jobid == cd->jobid) { + /* found it - tell the server to deregister */ + cd->active = true; + PMIx_server_deregister_nspace(jptr->nspace, tdcbfunc, cd); + PMIX_WAIT_FOR_COMPLETION(cd->active); + OBJ_RELEASE(cd); + /* now get rid of it from our list */ + opal_list_remove_item(&mca_pmix_pmix2x_component.jobids, &jptr->super); + OBJ_RELEASE(jptr); + return; + } + } + /* must release the caller */ + tdcbfunc(PMIX_ERR_NOT_FOUND, cd); +} + +void pmix2x_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix2x_threadshift_t *cd; + + /* we must threadshift this request as it touches + * shared lists of objects */ + cd = OBJ_NEW(pmix2x_threadshift_t); + cd->jobid = jobid; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + if (NULL == cbfunc) { + _dereg_nspace(0, 0, cd); + } else { + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _dereg_nspace, cd); + event_active(&cd->ev, EV_WRITE, 1); + } +} + +int pmix2x_server_register_client(const opal_process_name_t *proc, + uid_t uid, gid_t gid, + void *server_object, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_status_t rc; + pmix_proc_t p; + pmix2x_opcaddy_t op; + + /* convert the jobid */ + (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid); + p.rank = proc->vpid; + + OBJ_CONSTRUCT(&op, pmix2x_opcaddy_t); + op.active = true; + rc = PMIx_server_register_client(&p, uid, gid, server_object, + opcbfunc, (void*)&op); + if (PMIX_SUCCESS == rc) { + PMIX_WAIT_FOR_COMPLETION(op.active); + rc = op.status; + } + OBJ_DESTRUCT(&op); + return pmix2x_convert_rc(rc); +} + +static void _dereg_client(int sd, short args, void *cbdata) +{ + pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata; + opal_pmix2x_jobid_trkr_t *jptr; + pmix_proc_t p; + + /* if we don't already have it, we can ignore this */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_pmix2x_component.jobids, opal_pmix2x_jobid_trkr_t) { + if (jptr->jobid == cd->source->jobid) { + /* found it - tell the server to deregister */ + (void)strncpy(p.nspace, jptr->nspace, PMIX_MAX_NSLEN); + p.rank = cd->source->vpid; + cd->active = true; + PMIx_server_deregister_client(&p, tdcbfunc, (void*)cd); + PMIX_WAIT_FOR_COMPLETION(cd->active); + break; + } + } + OBJ_RELEASE(cd); +} + +/* tell the local PMIx server to cleanup this client as it is + * done executing */ +void pmix2x_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix2x_threadshift_t *cd; + + /* we must threadshift this request as we might not be in an event + * and we are going to access framework-global lists/objects */ + cd = OBJ_NEW(pmix2x_threadshift_t); + cd->source = proc; + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + if (NULL == cbfunc) { + _dereg_client(0, 0, cd); + } else { + event_assign(&cd->ev, opal_pmix_base.evbase, + -1, EV_WRITE, _dereg_client, cd); + event_active(&cd->ev, EV_WRITE, 1); + } +} + +/* have the local PMIx server setup the environment for this client */ +int pmix2x_server_setup_fork(const opal_process_name_t *proc, char ***env) +{ + pmix_status_t rc; + pmix_proc_t p; + + /* convert the jobid */ + (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid); + p.rank = proc->vpid; + + rc = PMIx_server_setup_fork(&p, env); + return pmix2x_convert_rc(rc); +} + +/* this is the call back up from the embedded PMIx server that + * will contain the returned data. Note that the embedded server + * "owns" the data and will free it upon return from this function */ +static void dmdx_response(pmix_status_t status, char *data, size_t sz, void *cbdata) +{ + int rc; + pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata; + + rc = pmix2x_convert_rc(status); + if (NULL != op->mdxcbfunc) { + op->mdxcbfunc(rc, data, sz, op->cbdata, NULL, NULL); + } + OBJ_RELEASE(op); +} + +/* request modex data for a local proc from the PMIx server */ +int pmix2x_server_dmodex(const opal_process_name_t *proc, + opal_pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + pmix2x_opcaddy_t *op; + pmix_status_t rc; + + /* setup the caddy */ + op = OBJ_NEW(pmix2x_opcaddy_t); + op->mdxcbfunc = cbfunc; + op->cbdata = cbdata; + + /* convert the jobid */ + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, proc->jobid); + op->p.rank = proc->vpid; + + /* find the internally-cached data for this proc */ + rc = PMIx_server_dmodex_request(&op->p, dmdx_response, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return pmix2x_convert_rc(rc); +} + +/* tell the PMIx server to notify its local clients of an event */ +int pmix2x_server_notify_event(int status, + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + opal_value_t *kv; + pmix_info_t *pinfo; + size_t sz, n; + pmix_status_t rc; + pmix2x_opcaddy_t *op; + + /* convert the list to an array of pmix_info_t */ + if (NULL != info) { + sz = opal_list_get_size(info); + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix2x_value_load(&pinfo[n].value, kv); + } + } else { + sz = 0; + pinfo = NULL; + } + /* setup the caddy */ + op = OBJ_NEW(pmix2x_opcaddy_t); + op->info = pinfo; + op->sz = sz; + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + /* convert the jobid */ + if (NULL == source) { + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, OPAL_JOBID_INVALID); + op->p.rank = OPAL_VPID_INVALID; + } else { + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid); + op->p.rank = source->vpid; + } + + + rc = pmix2x_convert_opalrc(status); + /* the range is irrelevant here as the server is passing + * the event down to its local clients */ + rc = PMIx_Notify_event(rc, &op->p, PMIX_RANGE_LOCAL, + pinfo, sz, opcbfunc, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return pmix2x_convert_rc(rc); +} diff --git a/opal/mca/pmix/pmix_server.h b/opal/mca/pmix/pmix_server.h index e7e73ff8c3..ca51dd04a6 100644 --- a/opal/mca/pmix/pmix_server.h +++ b/opal/mca/pmix/pmix_server.h @@ -32,7 +32,9 @@ BEGIN_C_DECLS /* Notify the host server that a client connected to us */ typedef int (*opal_pmix_server_client_connected_fn_t)(opal_process_name_t *proc, - void* server_object); + void* server_object, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); /* Notify the host server that a client called pmix.finalize - note * that the client will be in a blocked state until the host server @@ -192,7 +194,7 @@ typedef void (*opal_pmix_connection_cbfunc_t)(int incoming_sd); * numbers of local clients such as occur when running on large * SMPs. The host server listener is required to call accept * on the incoming connection request, and then passing the - * resulting soct to the provided cbfunc. A NULL for this function + * resulting socket to the provided cbfunc. A NULL for this function * will cause the internal PMIx server to spawn its own listener * thread */ typedef int (*opal_pmix_server_listener_fn_t)(int listening_sd, diff --git a/opal/mca/pmix/pmix_types.h b/opal/mca/pmix/pmix_types.h index d81577824d..2ffa0bdd02 100644 --- a/opal/mca/pmix/pmix_types.h +++ b/opal/mca/pmix/pmix_types.h @@ -17,6 +17,19 @@ BEGIN_C_DECLS +/* define a value for requests for job-level data + * where the info itself isn't associated with any + * specific rank, or when a request involves + * a rank that isn't known - e.g., when someone requests + * info thru one of the legacy interfaces where the rank + * is typically encoded into the key itself since there is + * no rank parameter in the API itself */ +#define OPAL_PMIX_RANK_UNDEF INT32_MAX +/* define a value to indicate that the user wants the + * data for the given key from every rank that posted + * that key */ +#define OPAL_PMIX_RANK_WILDCARD INT32_MAX-1 + /* define a set of "standard" attributes that can * be queried. Implementations (and users) are free to extend as * desired, so the get functions need to be capable @@ -29,33 +42,36 @@ BEGIN_C_DECLS #define OPAL_PMIX_ATTR_UNDEF NULL /* identification attributes */ -#define OPAL_PMIX_USERID "pmix.euid" // (uint32_t) effective user id -#define OPAL_PMIX_GRPID "pmix.egid" // (uint32_t) effective group id +#define OPAL_PMIX_USERID "pmix.euid" // (uint32_t) effective user id +#define OPAL_PMIX_GRPID "pmix.egid" // (uint32_t) effective group id + +/* attributes for the rendezvous socket */ +#define OPAL_PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid) /* general proc-level attributes */ -#define OPAL_PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch -#define OPAL_PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc -#define OPAL_PMIX_SPAWNED "pmix.spawned" // (bool) true if this proc resulted from a call to PMIx_Spawn -#define OPAL_PMIX_ARCH "pmix.arch" // (uint32_t) datatype architecture flag +#define OPAL_PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch +#define OPAL_PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc +#define OPAL_PMIX_SPAWNED "pmix.spawned" // (bool) true if this proc resulted from a call to PMIx_Spawn +#define OPAL_PMIX_ARCH "pmix.arch" // (uint32_t) datatype architecture flag /* scratch directory locations for use by applications */ -#define OPAL_PMIX_TMPDIR "pmix.tmpdir" // (char*) top-level tmp dir assigned to session -#define OPAL_PMIX_NSDIR "pmix.nsdir" // (char*) sub-tmpdir assigned to namespace -#define OPAL_PMIX_PROCDIR "pmix.pdir" // (char*) sub-nsdir assigned to proc +#define OPAL_PMIX_TMPDIR "pmix.tmpdir" // (char*) top-level tmp dir assigned to session +#define OPAL_PMIX_NSDIR "pmix.nsdir" // (char*) sub-tmpdir assigned to namespace +#define OPAL_PMIX_PROCDIR "pmix.pdir" // (char*) sub-nsdir assigned to proc /* information about relative ranks as assigned by the RM */ -#define OPAL_PMIX_JOBID "pmix.jobid" // (uint32_t) jobid assigned by scheduler -#define OPAL_PMIX_APPNUM "pmix.appnum" // (uint32_t) app number within the job -#define OPAL_PMIX_RANK "pmix.rank" // (uint32_t) process rank within the job -#define OPAL_PMIX_GLOBAL_RANK "pmix.grank" // (uint32_t) rank spanning across all jobs in this session -#define OPAL_PMIX_UNIV_RANK "pmix.grank" // (uint32_t) synonym for global_rank -#define OPAL_PMIX_APP_RANK "pmix.apprank" // (uint32_t) rank within this app -#define OPAL_PMIX_NPROC_OFFSET "pmix.offset" // (uint32_t) starting global rank of this job -#define OPAL_PMIX_LOCAL_RANK "pmix.lrank" // (uint16_t) rank on this node within this job -#define OPAL_PMIX_NODE_RANK "pmix.nrank" // (uint16_t) rank on this node spanning all jobs -#define OPAL_PMIX_LOCALLDR "pmix.lldr" // (uint64_t) opal_identifier of lowest rank on this node within this job -#define OPAL_PMIX_APPLDR "pmix.aldr" // (uint32_t) lowest rank in this app within this job -#define OPAL_PMIX_LOCALITY "pmix.loc" // (uint16_t) relative locality of two procs +#define OPAL_PMIX_JOBID "pmix.jobid" // (uint32_t) jobid assigned by scheduler +#define OPAL_PMIX_APPNUM "pmix.appnum" // (uint32_t) app number within the job +#define OPAL_PMIX_RANK "pmix.rank" // (uint32_t) process rank within the job +#define OPAL_PMIX_GLOBAL_RANK "pmix.grank" // (uint32_t) rank spanning across all jobs in this session +#define OPAL_PMIX_UNIV_RANK "pmix.grank" // (uint32_t) synonym for global_rank +#define OPAL_PMIX_APP_RANK "pmix.apprank" // (uint32_t) rank within this app +#define OPAL_PMIX_NPROC_OFFSET "pmix.offset" // (uint32_t) starting global rank of this job +#define OPAL_PMIX_LOCAL_RANK "pmix.lrank" // (uint16_t) rank on this node within this job +#define OPAL_PMIX_NODE_RANK "pmix.nrank" // (uint16_t) rank on this node spanning all jobs +#define OPAL_PMIX_LOCALLDR "pmix.lldr" // (uint64_t) opal_identifier of lowest rank on this node within this job +#define OPAL_PMIX_APPLDR "pmix.aldr" // (uint32_t) lowest rank in this app within this job +#define OPAL_PMIX_LOCALITY "pmix.loc" // (uint16_t) relative locality of two procs /* proc location-related info */ /* For PMIX_HOSTNAME, three use-cases exist for PMIx_Get: * @@ -70,94 +86,88 @@ BEGIN_C_DECLS * (c) Specifying a namespace and a rank will return the name of the * host this proc is on */ -#define OPAL_PMIX_HOSTNAME "pmix.hname" // (char*) see above comment -#define OPAL_PMIX_NODEID "pmix.nodeid" // (uint32_t) node identifier -#define OPAL_PMIX_LOCAL_PEERS "pmix.lpeers" // (char*) comma-delimited string of ranks on this node within the specified nspace -#define OPAL_PMIX_LOCAL_CPUSETS "pmix.lcpus" // (char*) colon-delimited cpusets of local peers within the specified nspace -#define OPAL_PMIX_PROC_URI "pmix.puri" // (char*) URI containing contact info for proc +#define OPAL_PMIX_HOSTNAME "pmix.hname" // (char*) see above comment +#define OPAL_PMIX_NODEID "pmix.nodeid" // (uint32_t) node identifier +#define OPAL_PMIX_LOCAL_PEERS "pmix.lpeers" // (char*) comma-delimited string of ranks on this node within the specified nspace +#define OPAL_PMIX_LOCAL_CPUSETS "pmix.lcpus" // (char*) colon-delimited cpusets of local peers within the specified nspace +#define OPAL_PMIX_PROC_URI "pmix.puri" // (char*) URI containing contact info for proc /* size info */ -#define OPAL_PMIX_UNIV_SIZE "pmix.univ.size" // (uint32_t) #procs in this nspace -#define OPAL_PMIX_JOB_SIZE "pmix.job.size" // (uint32_t) #procs in this job -#define OPAL_PMIX_APP_SIZE "pmix.app.size" // (uint32_t) #procs in this app -#define OPAL_PMIX_LOCAL_SIZE "pmix.local.size" // (uint32_t) #procs in this job on this node -#define OPAL_PMIX_NODE_SIZE "pmix.node.size" // (uint32_t) #procs across all jobs on this node -#define OPAL_PMIX_MAX_PROCS "pmix.max.size" // (uint32_t) max #procs for this job +#define OPAL_PMIX_UNIV_SIZE "pmix.univ.size" // (uint32_t) #procs in this nspace +#define OPAL_PMIX_JOB_SIZE "pmix.job.size" // (uint32_t) #procs in this job +#define OPAL_PMIX_APP_SIZE "pmix.app.size" // (uint32_t) #procs in this app +#define OPAL_PMIX_LOCAL_SIZE "pmix.local.size" // (uint32_t) #procs in this job on this node +#define OPAL_PMIX_NODE_SIZE "pmix.node.size" // (uint32_t) #procs across all jobs on this node +#define OPAL_PMIX_MAX_PROCS "pmix.max.size" // (uint32_t) max #procs for this job /* topology info */ -#define OPAL_PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology -#define OPAL_PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology -#define OPAL_PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job -#define OPAL_PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object - -/* fault tolerance-related info */ -#define OPAL_PMIX_TERMINATE_SESSION "pmix.term.sess" // (bool) RM intends to terminate session -#define OPAL_PMIX_TERMINATE_JOB "pmix.term.job" // (bool) RM intends to terminate this job -#define OPAL_PMIX_TERMINATE_NODE "pmix.term.node" // (bool) RM intends to terminate all procs on this node -#define OPAL_PMIX_TERMINATE_PROC "pmix.term.proc" // (bool) RM intends to terminate just this process -#define OPAL_PMIX_ERROR_ACTION_TIMEOUT "pmix.err.timeout" // (int) time in sec before RM will execute error response +#define OPAL_PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology +#define OPAL_PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology +#define OPAL_PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job +#define OPAL_PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object /* request-related info */ -#define OPAL_PMIX_COLLECT_DATA "pmix.collect" // (bool) collect data and return it at the end of the operation -#define OPAL_PMIX_TIMEOUT "pmix.timeout" // (int) time in sec before specified operation should time out -#define OPAL_PMIX_WAIT "pmix.wait" // (int) caller requests that the server wait until at least the specified - // #values are found (0 => all and is the default) -#define OPAL_PMIX_COLLECTIVE_ALGO "pmix.calgo" // (char*) comma-delimited list of algorithms to use for collective -#define OPAL_PMIX_COLLECTIVE_ALGO_REQD "pmix.calreqd" // (bool) if true, indicates that the requested choice of algo is mandatory -#define OPAL_PMIX_NOTIFY_COMPLETION "pmix.notecomp" // (bool) notify parent process upon termination of child job -#define OPAL_PMIX_RANGE "pmix.range" // (int) opal_pmix_data_range_t value for calls to publish/lookup/unpublish -#define OPAL_PMIX_PERSISTENCE "pmix.persist" // (int) opal_pmix_persistence_t value for calls to publish -#define OPAL_PMIX_OPTIONAL "pmix.optional" // (bool) look only in the immediate data store for the requested value - do - // not request data from the server if not found +#define OPAL_PMIX_COLLECT_DATA "pmix.collect" // (bool) collect data and return it at the end of the operation +#define OPAL_PMIX_TIMEOUT "pmix.timeout" // (int) time in sec before specified operation should time out +#define OPAL_PMIX_WAIT "pmix.wait" // (int) caller requests that the server wait until at least the specified + // #values are found (0 => all and is the default) +#define OPAL_PMIX_COLLECTIVE_ALGO "pmix.calgo" // (char*) comma-delimited list of algorithms to use for collective +#define OPAL_PMIX_COLLECTIVE_ALGO_REQD "pmix.calreqd" // (bool) if true, indicates that the requested choice of algo is mandatory +#define OPAL_PMIX_NOTIFY_COMPLETION "pmix.notecomp" // (bool) notify parent process upon termination of child job +#define OPAL_PMIX_RANGE "pmix.range" // (int) opal_pmix_data_range_t value for calls to publish/lookup/unpublish +#define OPAL_PMIX_PERSISTENCE "pmix.persist" // (int) opal_pmix_persistence_t value for calls to publish +#define OPAL_PMIX_OPTIONAL "pmix.optional" // (bool) look only in the immediate data store for the requested value - do + // not request data from the server if not found +#define OPAL_PMIX_EMBED_BARRIER "pmix.embed.barrier" // (bool) execute a blocking fence operation before executing the + // specified operation /* attribute used by host server to pass data to the server convenience library - the * data will then be parsed and provided to the local clients */ -#define OPAL_PMIX_PROC_DATA "pmix.pdata" // (pmix_value_array_t) starts with rank, then contains more data -#define OPAL_PMIX_NODE_MAP "pmix.nmap" // (char*) regex of nodes containing procs for this job -#define OPAL_PMIX_PROC_MAP "pmix.pmap" // (char*) regex describing procs on each node within this job +#define OPAL_PMIX_PROC_DATA "pmix.pdata" // (pmix_value_array_t) starts with rank, then contains more data +#define OPAL_PMIX_NODE_MAP "pmix.nmap" // (char*) regex of nodes containing procs for this job +#define OPAL_PMIX_PROC_MAP "pmix.pmap" // (char*) regex describing procs on each node within this job /* attributes used internally to communicate data from the server to the client */ -#define OPAL_PMIX_PROC_BLOB "pmix.pblob" // (pmix_byte_object_t) packed blob of process data -#define OPAL_PMIX_MAP_BLOB "pmix.mblob" // (pmix_byte_object_t) packed blob of process location +#define OPAL_PMIX_PROC_BLOB "pmix.pblob" // (pmix_byte_object_t) packed blob of process data +#define OPAL_PMIX_MAP_BLOB "pmix.mblob" // (pmix_byte_object_t) packed blob of process location /* error handler registration and notification info keys */ -#define OPAL_PMIX_ERROR_NAME "pmix.errname" /* enum pmix_status_t specific error to be notified */ -#define OPAL_PMIX_ERROR_GROUP_COMM "pmix.errgroup.comm" /* bool - set true to get comm errors notification */ -#define OPAL_PMIX_ERROR_GROUP_ABORT "pmix.errgroup.abort" /* bool -set true to get abort errors notification */ -#define OPAL_PMIX_ERROR_GROUP_MIGRATE "pmix.errgroup.migrate" /* bool -set true to get migrate errors notification */ -#define OPAL_PMIX_ERROR_GROUP_RESOURCE "pmix.errgroup.resource" /* bool -set true to get resource errors notification */ -#define OPAL_PMIX_ERROR_GROUP_SPAWN "pmix.errgroup.spawn" /* bool - set true to get spawn errors notification */ -#define OPAL_PMIX_ERROR_GROUP_NODE "pmix.errgroup.node" /* bool -set true to get node status errors */ -#define OPAL_PMIX_ERROR_GROUP_LOCAL "pmix.errgroup.local" /* bool set true to get local errors */ -#define OPAL_PMIX_ERROR_GROUP_GENERAL "pmix.errgroup.gen" /* bool set true to get notified af generic errors */ +#define OPAL_PMIX_EVENT_HDLR_NAME "pmix.evname" // (char*) string name identifying this handler +#define OPAL_PMIX_EVENT_JOB_LEVEL "pmix.evjob" // (bool) register for job-specific events only +#define OPAL_PMIX_EVENT_ENVIRO_LEVEL "pmix.evenv" // (bool) register for environment events only +#define OPAL_PMIX_EVENT_ORDER_PREPEND "pmix.evprepend" // (bool) prepend this handler to the precedence list +#define OPAL_PMIX_EVENT_CUSTOM_RANGE "pmix.evrange" // (pmix_proc_t*) array of pmix_proc_t defining range of event notification +#define OPAL_PMIX_EVENT_AFFECTED_PROCS "pmix.evaffected" // (pmix_proc_t*) array of pmix_proc_t defining affected procs +#define OPAL_PMIX_EVENT_NON_DEFAULT "opal.evnondef" // (bool) event is not to be delivered to default event handlers +/* fault tolerance-related events */ +#define OPAL_PMIX_EVENT_TERMINATE_SESSION "pmix.evterm.sess" // (bool) RM intends to terminate session +#define OPAL_PMIX_EVENT_TERMINATE_JOB "pmix.evterm.job" // (bool) RM intends to terminate this job +#define OPAL_PMIX_EVENT_TERMINATE_NODE "pmix.evterm.node" // (bool) RM intends to terminate all procs on this node +#define OPAL_PMIX_EVENT_TERMINATE_PROC "pmix.evterm.proc" // (bool) RM intends to terminate just this process +#define OPAL_PMIX_EVENT_ACTION_TIMEOUT "pmix.evtimeout" // (int) time in sec before RM will execute error response -/* error notification keys */ -#define OPAL_PMIX_ERROR_SCOPE "pmix.errscope" /* int (enum pmix_scope_t) scope of error notification*/ -#define OPAL_PMIX_ERROR_NODE_NAME "pmix.errnode.name" /* name of the node that is in error or which reported the error.*/ -#define OPAL_PMIX_ERROR_SEVERITY "pmix.errseverity" /* the severity of the notified (reported) error */ /* attributes used to describe "spawm" attributes */ -#define OPAL_PMIX_PERSONALITY "pmix.pers" // (char*) name of personality to use -#define OPAL_PMIX_HOST "pmix.host" // (char*) comma-delimited list of hosts to use for spawned procs -#define OPAL_PMIX_HOSTFILE "pmix.hostfile" // (char*) hostfile to use for spawned procs -#define OPAL_PMIX_ADD_HOST "pmix.addhost" // (char*) comma-delimited list of hosts to add to allocation -#define OPAL_PMIX_ADD_HOSTFILE "pmix.addhostfile" // (char*) hostfile to add to existing allocation -#define OPAL_PMIX_PREFIX "pmix.prefix" // (char*) prefix to use for starting spawned procs -#define OPAL_PMIX_WDIR "pmix.wdir" // (char*) working directory for spawned procs -#define OPAL_PMIX_MAPPER "pmix.mapper" // (char*) mapper to use for placing spawned procs -#define OPAL_PMIX_DISPLAY_MAP "pmix.dispmap" // (bool) display process map upon spawn -#define OPAL_PMIX_PPR "pmix.ppr" // (char*) #procs to spawn on each identified resource -#define OPAL_PMIX_MAPBY "pmix.mapby" // (char*) mapping policy -#define OPAL_PMIX_RANKBY "pmix.rankby" // (char*) ranking policy -#define OPAL_PMIX_BINDTO "pmix.bindto" // (char*) binding policy -#define OPAL_PMIX_PRELOAD_BIN "pmix.preloadbin" // (bool) preload binaries -#define OPAL_PMIX_PRELOAD_FILES "pmix.preloadfiles" // (char*) comma-delimited list of files to pre-position -#define OPAL_PMIX_NON_PMI "pmix.nonpmi" // (bool) spawned procs will not call PMIx_Init -#define OPAL_PMIX_STDIN_TGT "pmix.stdin" // (uint32_t) spawned proc rank that is to receive stdin +#define OPAL_PMIX_PERSONALITY "pmix.pers" // (char*) name of personality to use +#define OPAL_PMIX_HOST "pmix.host" // (char*) comma-delimited list of hosts to use for spawned procs +#define OPAL_PMIX_HOSTFILE "pmix.hostfile" // (char*) hostfile to use for spawned procs +#define OPAL_PMIX_ADD_HOST "pmix.addhost" // (char*) comma-delimited list of hosts to add to allocation +#define OPAL_PMIX_ADD_HOSTFILE "pmix.addhostfile" // (char*) hostfile to add to existing allocation +#define OPAL_PMIX_PREFIX "pmix.prefix" // (char*) prefix to use for starting spawned procs +#define OPAL_PMIX_WDIR "pmix.wdir" // (char*) working directory for spawned procs +#define OPAL_PMIX_MAPPER "pmix.mapper" // (char*) mapper to use for placing spawned procs +#define OPAL_PMIX_DISPLAY_MAP "pmix.dispmap" // (bool) display process map upon spawn +#define OPAL_PMIX_PPR "pmix.ppr" // (char*) #procs to spawn on each identified resource +#define OPAL_PMIX_MAPBY "pmix.mapby" // (char*) mapping policy +#define OPAL_PMIX_RANKBY "pmix.rankby" // (char*) ranking policy +#define OPAL_PMIX_BINDTO "pmix.bindto" // (char*) binding policy +#define OPAL_PMIX_PRELOAD_BIN "pmix.preloadbin" // (bool) preload binaries +#define OPAL_PMIX_PRELOAD_FILES "pmix.preloadfiles" // (char*) comma-delimited list of files to pre-position +#define OPAL_PMIX_NON_PMI "pmix.nonpmi" // (bool) spawned procs will not call PMIx_Init +#define OPAL_PMIX_STDIN_TGT "pmix.stdin" // (uint32_t) spawned proc rank that is to receive stdin -/* define a scope for data "put" by PMI per the following - maintain - * consistent order with the PMIx distro : +/* define a scope for data "put" by PMI per the following: * * OPAL_PMI_LOCAL - the data is intended only for other application * processes on the same node. Data marked in this way @@ -176,17 +186,18 @@ typedef enum { OPAL_PMIX_GLOBAL } opal_pmix_scope_t; -/* define a range for data "published" by PMI - maintain - * consistent order with the PMIx distro */ +/* define a range for data "published" by PMI */ #define OPAL_PMIX_DATA_RANGE OPAL_UINT typedef enum { - OPAL_PMIX_DATA_RANGE_UNDEF = 0, - OPAL_PMIX_NAMESPACE, // data is available to procs in the same nspace only - OPAL_PMIX_SESSION // data available to all jobs in this session + OPAL_PMIX_RANGE_UNDEF = 0, + OPAL_PMIX_RANGE_LOCAL, // available on local node only + OPAL_PMIX_RANGE_NAMESPACE, // data is available to procs in the same nspace only + OPAL_PMIX_RANGE_SESSION, // data available to all procs in session + OPAL_PMIX_RANGE_GLOBAL, // data available to all procs + OPAL_PMIX_RANGE_CUSTOM // range is specified in a opal_value_t } opal_pmix_data_range_t; -/* define a "persistence" policy for data published by clients - maintain - * consistent order with the PMIx distro */ +/* define a "persistence" policy for data published by clients */ typedef enum { OPAL_PMIX_PERSIST_INDEF = 0, // retain until specifically deleted OPAL_PMIX_PERSIST_FIRST_READ, // delete upon first access @@ -274,32 +285,38 @@ typedef void (*opal_pmix_lookup_cbfunc_t)(int status, opal_list_t *data, void *cbdata); -/* define a callback function for the errhandler. Upon receipt of an - * error notification, the active module will execute the specified notification +/* define a callback function by which event handlers can notify + * us that they have completed their action, and pass along any + * further information for subsequent handlers */ +typedef void (*opal_pmix_notification_complete_fn_t)(int status, opal_list_t *results, + opal_pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata); + +/* define a callback function for the evhandler. Upon receipt of an + * event notification, the active module will execute the specified notification * callback function, providing: * * status - the error that occurred - * procs - the nspace and ranks of the affected processes. A NULL - * value indicates that the error occurred in the module - * library within this process itself + * source - identity of the proc that generated the event * info - any additional info provided regarding the error. - * cbfunc - callback function to execute when the errhandler is + * results - any info from prior event handlers + * cbfunc - callback function to execute when the evhandler is * finished with the provided data so it can be released * cbdata - pointer to be returned in cbfunc * * Note that different resource managers may provide differing levels - * of support for error notification to application processes. Thus, the - * info list may be NULL or may contain detailed information of the error. + * of support for event notification to application processes. Thus, the + * info list may be NULL or may contain detailed information of the event. * It is the responsibility of the application to parse any provided info array * for defined key-values if it so desires. * * Possible uses of the opal_value_t list include: * * - for the RM to alert the process as to planned actions, such as - * to abort the session, in response to the reported error + * to abort the session, in response to the reported event * * - provide a timeout for alternative action to occur, such as for - * the application to request an alternate response to the error + * the application to request an alternate response to the event * * For example, the RM might alert the application to the failure of * a node that resulted in termination of several processes, and indicate @@ -316,19 +333,19 @@ typedef void (*opal_pmix_lookup_cbfunc_t)(int status, * On the server side, the notification function is used to inform the host * server of a detected error in the PMIx subsystem and/or client */ typedef void (*opal_pmix_notification_fn_t)(int status, - opal_list_t *procs, - opal_list_t *info, - opal_pmix_release_cbfunc_t cbfunc, + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, void *cbdata); -/* define a callback function for calls to register_errhandler. The - * status indicates if the request was successful or not, errhandler_ref is - * an integer reference assigned to the errhandler by PMIX, this reference +/* define a callback function for calls to register_evhandler. The + * status indicates if the request was successful or not, evhandler_ref is + * a size_t reference assigned to the evhandler by PMIX, this reference * must be used to deregister the err handler. A ptr to the original * cbdata is returned. */ -typedef void (*opal_pmix_errhandler_reg_cbfunc_t)(int status, - int errhandler_ref, - void *cbdata); +typedef void (*opal_pmix_evhandler_reg_cbfunc_t)(int status, + size_t evhandler_ref, + void *cbdata); /* define a callback function for calls to get_nb. The status * indicates if the requested data was found or not - a pointer to the diff --git a/opal/mca/pmix/s1/pmix_s1.c b/opal/mca/pmix/s1/pmix_s1.c index cf1a5a2c08..a7e1e660e0 100644 --- a/opal/mca/pmix/s1/pmix_s1.c +++ b/opal/mca/pmix/s1/pmix_s1.c @@ -69,8 +69,8 @@ const opal_pmix_base_module_t opal_pmix_s1_module = { .spawn = s1_spawn, .connect = s1_job_connect, .disconnect = s1_job_disconnect, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, + .register_evhandler = opal_pmix_base_register_handler, + .deregister_evhandler = opal_pmix_base_deregister_handler, .store_local = s1_store_local, .get_nspace = s1_get_nspace, .register_jobid = s1_register_jobid diff --git a/opal/mca/pmix/s2/pmix_s2.c b/opal/mca/pmix/s2/pmix_s2.c index 7994907969..a51147d17c 100644 --- a/opal/mca/pmix/s2/pmix_s2.c +++ b/opal/mca/pmix/s2/pmix_s2.c @@ -76,8 +76,8 @@ const opal_pmix_base_module_t opal_pmix_s2_module = { .spawn = s2_spawn, .connect = s2_job_connect, .disconnect = s2_job_disconnect, - .register_errhandler = opal_pmix_base_register_handler, - .deregister_errhandler = opal_pmix_base_deregister_handler, + .register_evhandler = opal_pmix_base_register_handler, + .deregister_evhandler = opal_pmix_base_deregister_handler, .store_local = s2_store_local, .get_nspace = s2_get_nspace, .register_jobid = s2_register_jobid diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index 72511e8ac6..d58729a096 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -15,7 +15,7 @@ * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2010-2015 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2013-2015 Intel, Inc. All rights reserved + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -256,6 +256,9 @@ opal_err2str(int errnum, const char **errmsg) case OPAL_ERR_DEBUGGER_RELEASE: retval = "Release debugger"; break; + case OPAL_ERR_HANDLERS_COMPLETE: + retval = "Event handler processing complete"; + break; default: retval = "UNRECOGNIZED"; } diff --git a/orte/include/orte/constants.h b/orte/include/orte/constants.h index 06871f03a0..c828dd26ad 100644 --- a/orte/include/orte/constants.h +++ b/orte/include/orte/constants.h @@ -86,6 +86,7 @@ enum { ORTE_ERR_CONNECTION_FAILED = OPAL_ERR_CONNECTION_FAILED, ORTE_ERR_AUTHENTICATION_FAILED = OPAL_ERR_AUTHENTICATION_FAILED, ORTE_ERR_COMM_FAILURE = OPAL_ERR_COMM_FAILURE, + ORTE_ERR_DEBUGGER_RELEASE = OPAL_ERR_DEBUGGER_RELEASE, /* error codes specific to ORTE - don't forget to update orte/util/error_strings.c when adding new error codes!! diff --git a/orte/mca/errmgr/default_app/errmgr_default_app.c b/orte/mca/errmgr/default_app/errmgr_default_app.c index c1ee58ee99..5b1d021f9f 100644 --- a/orte/mca/errmgr/default_app/errmgr_default_app.c +++ b/orte/mca/errmgr/default_app/errmgr_default_app.c @@ -9,7 +9,7 @@ * reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -27,7 +27,6 @@ #include "opal/util/output.h" #include "opal/dss/dss.h" -#include "opal/errhandler/opal_errhandler.h" #include "opal/mca/pmix/pmix.h" #include "orte/util/error_strings.h" @@ -70,37 +69,32 @@ }; static void proc_errors(int fd, short args, void *cbdata); -static void pmix_error(int error, opal_proc_t *proc, void *cbdata) -{ - OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base_framework.framework_output, - "%s errmgr:default_app: errhandler called", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); - /* push it into our event base */ - ORTE_ACTIVATE_PROC_STATE(ORTE_PROC_MY_NAME, ORTE_PROC_STATE_COMM_FAILED); -} +static size_t myerrhandle = SIZE_MAX; -static int myerrhandle = -1; - -static void register_cbfunc(int status, int errhndler, void *cbdata) +static void register_cbfunc(int status, size_t errhndler, void *cbdata) { myerrhandle = errhndler; } static void notify_cbfunc(int status, - opal_list_t *procs, - opal_list_t *info, - opal_pmix_release_cbfunc_t cbfunc, - void *cbdata) + const opal_process_name_t *source, + opal_list_t *info, opal_list_t *results, + opal_pmix_notification_complete_fn_t cbfunc, void *cbdata) { - if (NULL != cbfunc) { - cbfunc(cbdata); - } OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base_framework.framework_output, - "%s errmgr:default_app: pmix errhandler called", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); + "%s errmgr:default_app: pmix event handler called with status %s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + ORTE_ERROR_NAME(status))); + + /* let the caller know we processed this, but allow the + * chain to continue */ + if (NULL != cbfunc) { + cbfunc(ORTE_SUCCESS, NULL, NULL, NULL, cbdata); + } + /* push it into our event base */ - ORTE_ACTIVATE_PROC_STATE(ORTE_PROC_MY_NAME, ORTE_PROC_STATE_COMM_FAILED); + ORTE_ACTIVATE_PROC_STATE(ORTE_PROC_MY_NAME, status); } /************************ @@ -111,19 +105,17 @@ static void notify_cbfunc(int status, /* setup state machine to trap proc errors */ orte_state.add_proc_state(ORTE_PROC_STATE_ERROR, proc_errors, ORTE_ERROR_PRI); - /* register an errhandler */ - opal_register_errhandler(pmix_error, NULL); - - /* tie the default PMIx errhandler back to us */ - opal_pmix.register_errhandler(NULL, notify_cbfunc, register_cbfunc, NULL); + /* tie the default PMIx event handler back to us */ + opal_pmix.register_evhandler(NULL, NULL, notify_cbfunc, register_cbfunc, NULL); return ORTE_SUCCESS; } static int finalize(void) { - opal_deregister_errhandler(); - opal_pmix.deregister_errhandler(myerrhandle, NULL, NULL); + if (SIZE_MAX != myerrhandle) { + opal_pmix.deregister_evhandler(myerrhandle, NULL, NULL); + } return ORTE_SUCCESS; } diff --git a/orte/mca/state/base/state_base_fns.c b/orte/mca/state/base/state_base_fns.c index 184ef56d94..fa79db9d90 100644 --- a/orte/mca/state/base/state_base_fns.c +++ b/orte/mca/state/base/state_base_fns.c @@ -18,6 +18,7 @@ #include "opal/mca/pmix/pmix.h" #include "orte/runtime/orte_globals.h" +#include "orte/runtime/orte_wait.h" #include "orte/mca/errmgr/errmgr.h" #include "orte/mca/iof/iof.h" #include "orte/mca/rmaps/rmaps_types.h" @@ -524,7 +525,7 @@ void orte_state_base_track_procs(int fd, short argc, void *cbdata) pdata->state = state; if (ORTE_FLAG_TEST(pdata, ORTE_PROC_FLAG_LOCAL)) { /* tell the PMIx subsystem to cleanup this client */ - opal_pmix.server_deregister_client(proc); + opal_pmix.server_deregister_client(proc, NULL, NULL); /* Clean up the session directory as if we were the process * itself. This covers the case where the process died abnormally * and didn't cleanup its own session directory. @@ -609,7 +610,7 @@ void orte_state_base_check_all_complete(int fd, short args, void *cbdata) /* tell the PMIx server to release its data */ if (NULL != opal_pmix.server_deregister_nspace) { - opal_pmix.server_deregister_nspace(jdata->jobid); + opal_pmix.server_deregister_nspace(jdata->jobid, NULL, NULL); } i32ptr = &i32; diff --git a/orte/mca/state/dvm/state_dvm.c b/orte/mca/state/dvm/state_dvm.c index f61a585bb6..dc4ac4f8fc 100644 --- a/orte/mca/state/dvm/state_dvm.c +++ b/orte/mca/state/dvm/state_dvm.c @@ -30,6 +30,7 @@ #include "orte/util/nidmap.h" #include "orte/util/session_dir.h" #include "orte/runtime/orte_quit.h" +#include "orte/runtime/orte_wait.h" #include "orte/mca/state/state.h" #include "orte/mca/state/base/base.h" @@ -383,7 +384,7 @@ static void check_complete(int fd, short args, void *cbdata) /* tell the PMIx subsystem the job is complete */ if (NULL != opal_pmix.server_deregister_nspace) { - opal_pmix.server_deregister_nspace(jdata->jobid); + opal_pmix.server_deregister_nspace(jdata->jobid, NULL, NULL); } /* Release the resources used by this job. Since some errmgrs may want diff --git a/orte/orted/orted_submit.c b/orte/orted/orted_submit.c index d490082a43..28f6a371bb 100644 --- a/orte/orted/orted_submit.c +++ b/orte/orted/orted_submit.c @@ -61,6 +61,7 @@ #include "opal/mca/installdirs/installdirs.h" #include "opal/mca/hwloc/base/base.h" #include "opal/mca/base/base.h" +#include "opal/mca/pmix/pmix.h" #include "opal/util/argv.h" #include "opal/util/output.h" #include "opal/util/basename.h" @@ -2133,6 +2134,64 @@ static void orte_debugger_init_before_spawn(orte_job_t *jdata) static bool mpir_breakpoint_fired = false; +static void _send_notification(void) +{ + opal_buffer_t buf; + int status = OPAL_ERR_DEBUGGER_RELEASE; + orte_grpcomm_signature_t sig; + int rc; + opal_value_t kv, *kvptr; + + OBJ_CONSTRUCT(&buf, opal_buffer_t); + + /* pack the debugger_attached status */ + if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &status, 1, OPAL_INT))) { + ORTE_ERROR_LOG(rc); + OBJ_DESTRUCT(&buf); + return; + } + + /* the source is me */ + if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, ORTE_PROC_MY_NAME, 1, ORTE_NAME))) { + ORTE_ERROR_LOG(rc); + OBJ_DESTRUCT(&buf); + return; + } + + /* instruct that it is to go only to non-default evhandlers */ + status = 1; + if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &status, 1, OPAL_INT))) { + ORTE_ERROR_LOG(rc); + OBJ_DESTRUCT(&buf); + return; + } + OBJ_CONSTRUCT(&kv, opal_value_t); + kv.key = strdup(OPAL_PMIX_EVENT_NON_DEFAULT); + kv.type = OPAL_BOOL; + kv.data.flag = true; + kvptr = &kv; + if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &kvptr, 1, OPAL_VALUE))) { + ORTE_ERROR_LOG(rc); + OBJ_DESTRUCT(&kv); + OBJ_DESTRUCT(&buf); + return; + } + OBJ_DESTRUCT(&kv); + + /* xcast it to everyone */ + OBJ_CONSTRUCT(&sig, orte_grpcomm_signature_t); + sig.signature = (orte_process_name_t*)malloc(sizeof(orte_process_name_t)); + sig.signature[0].jobid = ORTE_PROC_MY_NAME->jobid; + sig.signature[0].vpid = ORTE_VPID_WILDCARD; + sig.sz = 1; + + if (ORTE_SUCCESS != (rc = orte_grpcomm.xcast(&sig, ORTE_RML_TAG_NOTIFICATION, &buf))) { + ORTE_ERROR_LOG(rc); + } + OBJ_DESTRUCT(&sig); + OBJ_DESTRUCT(&buf); +} + static void orte_debugger_dump(void) { int i; @@ -2282,8 +2341,6 @@ void orte_debugger_init_after_spawn(int fd, short event, void *cbdata) orte_proc_t *proc; orte_app_context_t *appctx; orte_vpid_t i, j; - opal_buffer_t *buf; - int rc; char **aliases, *aptr; /* if we couldn't get thru the mapper stage, we might @@ -2303,19 +2360,11 @@ void orte_debugger_init_after_spawn(int fd, short event, void *cbdata) /* trigger the debugger */ MPIR_Breakpoint(); - /* send a message to rank=0 to release it */ - if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, 0)) || - ORTE_PROC_STATE_UNTERMINATED < proc->state ) { - /* proc is already dead */ - return; - } - buf = OBJ_NEW(opal_buffer_t); /* don't need anything in this */ - if (0 > (rc = orte_rml.send_buffer_nb(&proc->name, buf, - ORTE_RML_TAG_DEBUGGER_RELEASE, - orte_rml_send_callback, NULL))) { - opal_output(0, "Error: could not send debugger release to MPI procs - error %s", ORTE_ERROR_NAME(rc)); - OBJ_RELEASE(buf); - } + opal_output_verbose(5, orte_debug_output, + "%s NOTIFYING DEBUGGER RELEASE", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)); + /* notify all procs that the debugger is ready */ + _send_notification(); } return; } @@ -2409,25 +2458,11 @@ void orte_debugger_init_after_spawn(int fd, short event, void *cbdata) /* trigger the debugger */ MPIR_Breakpoint(); - /* send a message to rank=0 to release it */ - if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, 0)) || - ORTE_PROC_STATE_UNTERMINATED < proc->state) { - /* proc is already dead or never registered with us (so we don't have - * contact info for him) - */ - return; - } opal_output_verbose(2, orte_debug_output, - "%s sending debugger release to %s", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), - ORTE_NAME_PRINT(&proc->name)); - buf = OBJ_NEW(opal_buffer_t); /* don't need anything in this */ - if (0 > (rc = orte_rml.send_buffer_nb(&proc->name, buf, - ORTE_RML_TAG_DEBUGGER_RELEASE, - orte_rml_send_callback, NULL))) { - opal_output(0, "Error: could not send debugger release to MPI procs - error %s", ORTE_ERROR_NAME(rc)); - OBJ_RELEASE(buf); - } + "%s NOTIFYING DEBUGGER RELEASE", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)); + /* notify all procs that the debugger is ready */ + _send_notification(); } else { /* if I am launching debugger daemons, then I need to do so now * that the job has been started and I know which nodes have @@ -2635,7 +2670,7 @@ static int process(char *orig_line, char *basename, opal_cmd_line_t *cmd_line, return ret; } -static void open_fifo (void) +static void open_fifo(void) { if (orte_debugger_attach_fd > 0) { close(orte_debugger_attach_fd); diff --git a/orte/orted/pmix/pmix_server_gen.c b/orte/orted/pmix/pmix_server_gen.c index b7bbdbe4eb..d0bcc32b30 100644 --- a/orte/orted/pmix/pmix_server_gen.c +++ b/orte/orted/pmix/pmix_server_gen.c @@ -76,15 +76,19 @@ static void _client_conn(int sd, short args, void *cbdata) ORTE_FLAG_SET(p, ORTE_PROC_FLAG_REG); ORTE_ACTIVATE_PROC_STATE(&p->name, ORTE_PROC_STATE_REGISTERED); } + if (NULL != cd->cbfunc) { + cd->cbfunc(OPAL_SUCCESS, cd->cbdata); + } OBJ_RELEASE(cd); } -int pmix_server_client_connected_fn(opal_process_name_t *proc, void *server_object) +int pmix_server_client_connected_fn(opal_process_name_t *proc, void *server_object, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { /* need to thread-shift this request as we are going * to access our global list of registered events */ ORTE_PMIX_THREADSHIFT(proc, server_object, OPAL_SUCCESS, NULL, - NULL, _client_conn, NULL, NULL); + NULL, _client_conn, cbfunc, cbdata); return ORTE_SUCCESS; } @@ -264,26 +268,22 @@ static void _notify_release(int status, void *cbdata) { orte_pmix_server_op_caddy_t *cd = (orte_pmix_server_op_caddy_t*)cbdata; - if (NULL != cd->procs) { - OPAL_LIST_RELEASE(cd->procs); - } - if (NULL != cd->eprocs) { - OPAL_LIST_RELEASE(cd->eprocs); - } if (NULL != cd->info) { OPAL_LIST_RELEASE(cd->info); } OBJ_RELEASE(cd); } + +/* someone has sent us an event that we need to distribute + * to our local clients */ void pmix_server_notify(int status, orte_process_name_t* sender, opal_buffer_t *buffer, orte_rml_tag_t tg, void *cbdata) { - opal_list_t *procs = NULL, *eprocs = NULL, *info = NULL; - int cnt, rc, ret, nprocs, n; - opal_namelist_t *nm; + int cnt, rc, ret, ninfo, n; opal_value_t *val; orte_pmix_server_op_caddy_t *cd; + orte_process_name_t source; opal_output_verbose(2, orte_pmix_server_globals.output, "%s Notification received", @@ -296,99 +296,45 @@ void pmix_server_notify(int status, orte_process_name_t* sender, return; } - /* unpack the target procs that are to be notified */ + /* unpack the source */ cnt = 1; - if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &nprocs, &cnt, OPAL_INT))) { + if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &source, &cnt, ORTE_NAME))) { ORTE_ERROR_LOG(rc); return; } - /* if any were provided, add them to the list */ - if (0 < nprocs) { - procs = OBJ_NEW(opal_list_t); - for (n=0; n < nprocs; n++) { - nm = OBJ_NEW(opal_namelist_t); - opal_list_append(procs, &nm->super); - if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &nm->name, &cnt, OPAL_NAME))) { - ORTE_ERROR_LOG(rc); - OPAL_LIST_RELEASE(procs); - return; - } - } - } - - /* unpack the procs that were impacted by the error */ - cnt = 1; - if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &nprocs, &cnt, OPAL_INT))) { - ORTE_ERROR_LOG(rc); - if (NULL != procs) { - OPAL_LIST_RELEASE(procs); - } - return; - } - - /* if any were provided, add them to the list */ - if (0 < nprocs) { - eprocs = OBJ_NEW(opal_list_t); - for (n=0; n < nprocs; n++) { - nm = OBJ_NEW(opal_namelist_t); - opal_list_append(eprocs, &nm->super); - if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &nm->name, &cnt, OPAL_NAME))) { - ORTE_ERROR_LOG(rc); - if (NULL != procs) { - OPAL_LIST_RELEASE(procs); - } - OPAL_LIST_RELEASE(eprocs); - return; - } - } - } - /* unpack the infos that were provided */ cnt = 1; - if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &nprocs, &cnt, OPAL_INT))) { + if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &ninfo, &cnt, OPAL_INT))) { ORTE_ERROR_LOG(rc); - if (NULL != procs) { - OPAL_LIST_RELEASE(procs); - } return; } /* if any were provided, add them to the list */ - if (0 < nprocs) { - info = OBJ_NEW(opal_list_t); - for (n=0; n < nprocs; n++) { + cd = OBJ_NEW(orte_pmix_server_op_caddy_t); + + if (0 < ninfo) { + cd->info = OBJ_NEW(opal_list_t); + for (n=0; n < ninfo; n++) { val = OBJ_NEW(opal_value_t); - opal_list_append(info, &val->super); if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &val, &cnt, OPAL_VALUE))) { ORTE_ERROR_LOG(rc); - if (NULL != procs) { - OPAL_LIST_RELEASE(procs); - } - if (NULL != eprocs) { - OPAL_LIST_RELEASE(eprocs); - } - OPAL_LIST_RELEASE(info); + OBJ_RELEASE(val); + OPAL_LIST_RELEASE(cd->info); + OBJ_RELEASE(cd); return; } + opal_list_append(cd->info, &val->super); } } - cd = OBJ_NEW(orte_pmix_server_op_caddy_t); - cd->procs = procs; - cd->eprocs = eprocs; - cd->info = info; - - if (OPAL_SUCCESS != (rc = opal_pmix.server_notify_error(ret, procs, eprocs, info, _notify_release, cd))) { + opal_output_verbose(2, orte_pmix_server_globals.output, + "%s NOTIFYING PMIX SERVER OF STATUS %d", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ret); + if (OPAL_SUCCESS != (rc = opal_pmix.server_notify_event(ret, &source, cd->info, _notify_release, cd))) { ORTE_ERROR_LOG(rc); - if (NULL != procs) { - OPAL_LIST_RELEASE(procs); - } - if (NULL != eprocs) { - OPAL_LIST_RELEASE(eprocs); - } - if (NULL != info) { - OPAL_LIST_RELEASE(info); + if (NULL != cd->info) { + OPAL_LIST_RELEASE(cd->info); } OBJ_RELEASE(cd); } diff --git a/orte/orted/pmix/pmix_server_internal.h b/orte/orted/pmix/pmix_server_internal.h index 39cd1d8db8..5b689089f6 100644 --- a/orte/orted/pmix/pmix_server_internal.h +++ b/orte/orted/pmix/pmix_server_internal.h @@ -152,7 +152,8 @@ OBJ_CLASS_DECLARATION(orte_pmix_mdx_caddy_t); } while(0); /* define the server module functions */ -extern int pmix_server_client_connected_fn(opal_process_name_t *proc, void* server_object); +extern int pmix_server_client_connected_fn(opal_process_name_t *proc, void* server_object, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); extern int pmix_server_client_finalized_fn(opal_process_name_t *proc, void* server_object, opal_pmix_op_cbfunc_t cbfunc, void *cbdata); extern int pmix_server_abort_fn(opal_process_name_t *proc, void *server_object, diff --git a/orte/orted/pmix/pmix_server_pub.c b/orte/orted/pmix/pmix_server_pub.c index 6ec4656346..5283cdef7d 100644 --- a/orte/orted/pmix/pmix_server_pub.c +++ b/orte/orted/pmix/pmix_server_pub.c @@ -93,7 +93,7 @@ int pmix_server_publish_fn(opal_process_name_t *proc, int rc; uint8_t cmd = ORTE_PMIX_PUBLISH_CMD; opal_value_t *iptr; - opal_pmix_data_range_t range = OPAL_PMIX_SESSION; + opal_pmix_data_range_t range = OPAL_PMIX_RANGE_SESSION; opal_pmix_persistence_t persist = OPAL_PMIX_PERSIST_APP; bool rset, pset; @@ -143,7 +143,7 @@ int pmix_server_publish_fn(opal_process_name_t *proc, } /* if the range is SESSION, then set the target to the global server */ - if (OPAL_PMIX_SESSION == range) { + if (OPAL_PMIX_RANGE_SESSION == range) { req->target = orte_pmix_server_globals.server; } else { req->target = *ORTE_PROC_MY_HNP; @@ -193,7 +193,7 @@ int pmix_server_lookup_fn(opal_process_name_t *proc, char **keys, uint8_t cmd = ORTE_PMIX_LOOKUP_CMD; int32_t nkeys, i; opal_value_t *iptr; - opal_pmix_data_range_t range = OPAL_PMIX_SESSION; + opal_pmix_data_range_t range = OPAL_PMIX_RANGE_SESSION; /* the list of info objects are directives for us - they include * things like timeout constraints, so there is no reason to @@ -227,7 +227,7 @@ int pmix_server_lookup_fn(opal_process_name_t *proc, char **keys, } /* if the range is SESSION, then set the target to the global server */ - if (OPAL_PMIX_SESSION == range) { + if (OPAL_PMIX_RANGE_SESSION == range) { req->target = orte_pmix_server_globals.server; } else { req->target = *ORTE_PROC_MY_HNP; @@ -280,7 +280,7 @@ int pmix_server_unpublish_fn(opal_process_name_t *proc, char **keys, uint8_t cmd = ORTE_PMIX_UNPUBLISH_CMD; uint32_t nkeys, n; opal_value_t *iptr; - opal_pmix_data_range_t range = OPAL_PMIX_SESSION; + opal_pmix_data_range_t range = OPAL_PMIX_RANGE_SESSION; /* create the caddy */ req = OBJ_NEW(pmix_server_req_t); @@ -317,7 +317,7 @@ int pmix_server_unpublish_fn(opal_process_name_t *proc, char **keys, } /* if the range is SESSION, then set the target to the global server */ - if (OPAL_PMIX_SESSION == range) { + if (OPAL_PMIX_RANGE_SESSION == range) { req->target = orte_pmix_server_globals.server; } else { req->target = *ORTE_PROC_MY_HNP; diff --git a/orte/orted/pmix/pmix_server_register_fns.c b/orte/orted/pmix/pmix_server_register_fns.c index ef8b9b57b0..f6f7cf6132 100644 --- a/orte/orted/pmix/pmix_server_register_fns.c +++ b/orte/orted/pmix/pmix_server_register_fns.c @@ -49,32 +49,6 @@ #include "pmix_server_internal.h" #include "pmix_server.h" -typedef struct { - volatile bool active; - opal_list_t *info; -} myxfer_t; - -static void opcbfunc(int status, void *cbdata) -{ - myxfer_t *p = (myxfer_t*)cbdata; - opal_list_t *lt = p->info; - opal_value_t *k1, *k2; - opal_list_t *pmap; - - if (NULL != lt) { - OPAL_LIST_FOREACH_SAFE(k1, k2, lt, opal_value_t) { - if (OPAL_PTR == k1->type) { - pmap = (opal_list_t*)k1->data.ptr; - OPAL_LIST_RELEASE(pmap); - } - opal_list_remove_item(lt, &k1->super); - OBJ_RELEASE(k1); - } - OBJ_RELEASE(lt); - } - p->active = false; -} - /* stuff proc attributes for sending back to a proc */ int orte_pmix_server_register_nspace(orte_job_t *jdata) { @@ -91,7 +65,6 @@ int orte_pmix_server_register_nspace(orte_job_t *jdata) orte_app_context_t *app; uid_t uid; gid_t gid; - myxfer_t *p; opal_output_verbose(2, orte_pmix_server_globals.output, "%s register nspace for %s", @@ -406,17 +379,11 @@ int orte_pmix_server_register_nspace(orte_job_t *jdata) orte_set_attribute(&jdata->attributes, ORTE_JOB_NSPACE_REGISTERED, ORTE_ATTR_LOCAL, NULL, OPAL_BOOL); /* pass it down */ - p = (myxfer_t*)malloc(sizeof(myxfer_t)); - p->active = true; - p->info = info; - if (OPAL_SUCCESS != opal_pmix.server_register_nspace(jdata->jobid, - jdata->num_local_procs, - info, opcbfunc, (void*)p)) { - OPAL_LIST_RELEASE(info); - return ORTE_ERROR; - } - ORTE_WAIT_FOR_COMPLETION(p->active); - free(p); + /* we are in an event, so no need to callback */ + rc = opal_pmix.server_register_nspace(jdata->jobid, + jdata->num_local_procs, + info, NULL, NULL); + OPAL_LIST_RELEASE(info); - return ORTE_SUCCESS; + return rc; } diff --git a/orte/test/mpi/abort.c b/orte/test/mpi/abort.c index 79448b9d80..65f17fddfd 100644 --- a/orte/test/mpi/abort.c +++ b/orte/test/mpi/abort.c @@ -7,6 +7,7 @@ #include #include +#include #include "mpi.h" diff --git a/orte/util/error_strings.c b/orte/util/error_strings.c index 2648e2fb22..42671206d1 100644 --- a/orte/util/error_strings.c +++ b/orte/util/error_strings.c @@ -234,6 +234,8 @@ int orte_err2str(int errnum, const char **errmsg) case ORTE_ERR_JOB_CANCELLED: retval = "Job cancelled"; break; + case ORTE_ERR_DEBUGGER_RELEASE: + retval = "Debugger release"; default: if (orte_report_silent_errors) { retval = "Unknown error";