diff --git a/ompi/mca/pml/v/Makefile.am b/ompi/mca/pml/v/Makefile.am index 48ea63f872..c90ce9289e 100644 --- a/ompi/mca/pml/v/Makefile.am +++ b/ompi/mca/pml/v/Makefile.am @@ -32,9 +32,13 @@ local_sources = \ pml_v_component.c \ pml_v_output.h \ pml_v_output.c \ - pml_v_protocol.h \ - pml_v_protocol_base.h \ - pml_v_protocol_base.c + vprotocol.h \ + base/base.h \ + base/vprotocol_base.c \ + base/vprotocol_base_select.c \ + base/vprotocol_base_parasite.c \ + base/vprotocol_base_request.h \ + base/vprotocol_base_request.c mcacomponentdir = $(libdir)/openmpi mcacomponent_LTLIBRARIES = $(component_install) diff --git a/ompi/mca/pml/v/base/base.h b/ompi/mca/pml/v/base/base.h new file mode 100644 index 0000000000..8653572c78 --- /dev/null +++ b/ompi/mca/pml/v/base/base.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef __INCLUDE_VPROTOCOL_BASE_H_ +#define __INCLUDE_VPROTOCOL_BASE_H_ + +#include "ompi_config.h" +#include "ompi/constants.h" +#include "opal/mca/mca.h" +#include "../vprotocol.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +OMPI_DECLSPEC int mca_vprotocol_base_open(void); +OMPI_DECLSPEC int mca_vprotocol_base_select(bool enable_progress_threads, + bool enable_mpi_threads); +OMPI_DECLSPEC int mca_vprotocol_base_parasite(void); +OMPI_DECLSPEC int mca_vprotocol_base_close(void); + +OMPI_DECLSPEC extern opal_list_t mca_vprotocol_base_components_available; +OMPI_DECLSPEC extern mca_vprotocol_base_component_t mca_vprotocol_component; +OMPI_DECLSPEC extern mca_vprotocol_base_module_t mca_vprotocol; + + +/* Macro for use in components that are of type vprotocol v1.0.0 + */ +#define MCA_VPROTOCOL_BASE_VERSION_1_0_0 \ + /* vprotocol v1.0 is chained to MCA v1.0 */ \ + MCA_BASE_VERSION_1_0_0, \ + /* vprotocol v1.0 */ \ + "vprotocol", 1, 0, 0 + +/* Macro to mark an invalid component version (0.0.0). Any component showing + * that version number will be ignored. + */ +#define MCA_VPROTOCOL_BASE_VERSION_0_0_0 \ + /* vprotocol v0.0 is chained to MCA v1.0 */ \ + MCA_BASE_VERSION_1_0_0, \ + /* vprotocol v0.0 */ \ + "vprotocol", 0, 0, 0 + +#define mca_vprotocol_base_selected() ( \ + 0 != mca_vprotocol_component.pmlm_version.mca_type_major_version \ +) + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif /* __INCLUDE_VPROTOCOL_BASE_H_ */ diff --git a/ompi/mca/pml/v/base/vprotocol_base.c b/ompi/mca/pml/v/base/vprotocol_base.c new file mode 100644 index 0000000000..e13ccea70d --- /dev/null +++ b/ompi/mca/pml/v/base/vprotocol_base.c @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "base.h" +#include "opal/mca/mca.h" +#include "opal/mca/base/base.h" +#include "static-components.h" + +opal_list_t mca_vprotocol_base_components_available; + +/* Load any vprotocol MCA component and call open function of all those + * components. + */ +int mca_vprotocol_base_open(void) +{ + OBJ_CONSTRUCT(&mca_vprotocol_base_components_available, opal_list_t); + return mca_base_components_open("vprotocol", 0, + mca_vprotocol_base_static_components, + &mca_vprotocol_base_components_available, + true); +} + +/* Close and unload any vprotocol MCA component loaded. + */ +int mca_vprotocol_base_close(void) +{ + int ret; + ret = mca_base_components_close(pml_v_output, + &mca_vprotocol_base_components_available, + NULL); + OBJ_DESTRUCT(&mca_vprotocol_base_components_available); + return ret; +} diff --git a/ompi/mca/pml/v/base/vprotocol_base_parasite.c b/ompi/mca/pml/v/base/vprotocol_base_parasite.c new file mode 100644 index 0000000000..839afadc69 --- /dev/null +++ b/ompi/mca/pml/v/base/vprotocol_base_parasite.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "base.h" +#include "vprotocol_base_request.h" + +int mca_vprotocol_base_parasite(void) { + if(mca_vprotocol.add_procs) + mca_pml.pml_add_procs = mca_vprotocol.add_procs; + if(mca_vprotocol.del_procs) + mca_pml.pml_del_procs = mca_vprotocol.del_procs; + if(mca_vprotocol.progress) + mca_pml.pml_progress = mca_vprotocol.progress; + if(mca_vprotocol.add_comm) + mca_pml.pml_add_comm = mca_vprotocol.add_comm; + if(mca_vprotocol.del_comm) + mca_pml.pml_del_comm = mca_vprotocol.del_comm; + if(mca_vprotocol.irecv_init) + mca_pml.pml_irecv_init = mca_vprotocol.irecv_init; + if(mca_vprotocol.irecv) + mca_pml.pml_irecv = mca_vprotocol.irecv; + if(mca_vprotocol.recv) + mca_pml.pml_recv = mca_vprotocol.recv; + if(mca_vprotocol.isend_init) + mca_pml.pml_isend_init = mca_vprotocol.isend_init; + if(mca_vprotocol.isend) + mca_pml.pml_isend = mca_vprotocol.isend; + if(mca_vprotocol.send) + mca_pml.pml_send = mca_vprotocol.send; + if(mca_vprotocol.iprobe) + mca_pml.pml_iprobe = mca_vprotocol.iprobe; + if(mca_vprotocol.probe) + mca_pml.pml_probe = mca_vprotocol.probe; + if(mca_vprotocol.start) + mca_pml.pml_start = mca_vprotocol.start; + if(mca_vprotocol.dump) + mca_pml.pml_dump = mca_vprotocol.dump; + + return mca_vprotocol_base_request_parasite(); +} diff --git a/ompi/mca/pml/v/base/vprotocol_base_request.c b/ompi/mca/pml/v/base/vprotocol_base_request.c new file mode 100644 index 0000000000..f704c2b685 --- /dev/null +++ b/ompi/mca/pml/v/base/vprotocol_base_request.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "vprotocol_base_request.h" + +int mca_vprotocol_base_request_parasite(void) +{ + int ret; + + if(mca_vprotocol.req_recv_class) + { + ompi_free_list_t pml_fl_save = mca_pml_base_recv_requests; + mca_pml_v.host_pml_req_recv_size = + pml_fl_save.fl_elem_class->cls_sizeof; + V_OUTPUT_VERBOSE(300, "req_rebuild: recv\tsize %lu+%lu\talignment=%lu", (unsigned long) mca_pml_v.host_pml_req_recv_size, (unsigned long) mca_vprotocol.req_recv_class->cls_sizeof, (unsigned long) pml_fl_save.fl_alignment); + mca_vprotocol.req_recv_class->cls_parent = + pml_fl_save.fl_elem_class; + mca_vprotocol.req_recv_class->cls_sizeof += + pml_fl_save.fl_elem_class->cls_sizeof; + /* rebuild the requests free list with the right size */ + OBJ_DESTRUCT(&mca_pml_base_recv_requests); + OBJ_CONSTRUCT(&mca_pml_base_recv_requests, ompi_free_list_t); + ret = ompi_free_list_init_ex(&mca_pml_base_recv_requests, + mca_vprotocol.req_recv_class->cls_sizeof, + pml_fl_save.fl_alignment, + mca_vprotocol.req_recv_class, + pml_fl_save.fl_num_allocated, + pml_fl_save.fl_max_to_alloc, + pml_fl_save.fl_num_per_alloc, + pml_fl_save.fl_mpool, + pml_fl_save.item_init, + pml_fl_save.ctx); + if(OMPI_SUCCESS != ret) return ret; + } + if(mca_vprotocol.req_send_class) + { + ompi_free_list_t pml_fl_save = mca_pml_base_send_requests; + mca_pml_v.host_pml_req_send_size = + pml_fl_save.fl_elem_class->cls_sizeof; + V_OUTPUT_VERBOSE(300, "req_rebuild: send\tsize %lu+%lu\talignment=%lu", (unsigned long) mca_pml_v.host_pml_req_send_size, (unsigned long) mca_vprotocol.req_send_class->cls_sizeof, (unsigned long) pml_fl_save.fl_alignment); + mca_vprotocol.req_send_class->cls_parent = + pml_fl_save.fl_elem_class; + mca_vprotocol.req_send_class->cls_sizeof += + pml_fl_save.fl_elem_class->cls_sizeof; + /* rebuild the requests free list with the right size */ + OBJ_DESTRUCT(&mca_pml_base_send_requests); + OBJ_CONSTRUCT(&mca_pml_base_send_requests, ompi_free_list_t); + ret = ompi_free_list_init_ex(&mca_pml_base_send_requests, + mca_vprotocol.req_send_class->cls_sizeof, + pml_fl_save.fl_alignment, + mca_vprotocol.req_send_class, + pml_fl_save.fl_num_allocated, + pml_fl_save.fl_max_to_alloc, + pml_fl_save.fl_num_per_alloc, + pml_fl_save.fl_mpool, + pml_fl_save.item_init, + pml_fl_save.ctx); + if(OMPI_SUCCESS != ret) return ret; + } + return OMPI_SUCCESS; + +} diff --git a/ompi/mca/pml/v/base/vprotocol_base_request.h b/ompi/mca/pml/v/base/vprotocol_base_request.h new file mode 100644 index 0000000000..60f993939f --- /dev/null +++ b/ompi/mca/pml/v/base/vprotocol_base_request.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef __INCLUDE_VPROTOCOL_REQUEST_H_ +#define __INCLUDE_VPROTOCOL_REQUEST_H_ + +#include "ompi_config.h" +#include "ompi/mca/pml/base/pml_base_request.h" +#include "../vprotocol.h" +#include "../pml_v.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + + +/** Rebuild the PML requests pools to make room for extra space at end of each + * request. + * The extra data is allocated in each requests so that it can hold instances + * of the req_recv_class and req_send_class fields of the + * mca_vprotocol_base_module_t. If those fields are NULL the requests are not + * recreated. + * @return OMPI_SUCCESS or failure status + */ +OMPI_DECLSPEC int mca_vprotocol_base_request_parasite(void); + + +/** Gives the actual address of the protocol specific part of a recv request. + * @param request (IN) the address of an ompi_request. + * @return address of the custom vprotocol data associated with the request. + */ +#define VPROTOCOL_RECV_REQ(req) \ + (((uintptr_t) req) + mca_pml_v.host_pml_req_recv_size) + +/** Gives the actual address of the protocol specific part of a send request. + * @param request (IN) the address of an ompi_request. + * @return address of the custom vprotocol data associated with the request. + */ +#define VPROTOCOL_SEND_REQ(req) \ + (((uintptr_t) req) + mca_pml_v.host_pml_req_send_size) + +/** Unified macro to get the actual address of the protocol specific part of + * an send - or - recv request. + * @param request (IN) the address of an ompi_request. + * @return address of the custom vprotocol data associated with the request. + */ +#define VPROTOCOL_REQ(req) ( \ + assert((MCA_PML_REQUEST_SEND == \ + ((mca_pml_base_request_t *) req)->req_type) || \ + (MCA_PML_REQUEST_RECV == \ + ((mca_pml_base_request_t *) req)->req_type)), \ + ((MCA_PML_REQUEST_SEND == ((mca_pml_base_request_t *) req)->req_type) \ + ? VPROTOCOL_SEND_REQ(req) \ + : VPROTOCOL_RECV_REQ(req) \ + ) \ +) + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif /* __INCLUDE_VPROTOCOL_REQUEST_H_ */ diff --git a/ompi/mca/pml/v/base/vprotocol_base_select.c b/ompi/mca/pml/v/base/vprotocol_base_select.c new file mode 100644 index 0000000000..93d5350423 --- /dev/null +++ b/ompi/mca/pml/v/base/vprotocol_base_select.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "base.h" +#include "opal/mca/mca.h" +#include "opal/mca/base/base.h" + +mca_vprotocol_base_module_t mca_vprotocol; +mca_vprotocol_base_component_t mca_vprotocol_component = { + {MCA_VPROTOCOL_BASE_VERSION_0_0_0} /* Initialized with invalid version */ +}; + +typedef struct opened_component_t { + opal_list_item_t super; + mca_vprotocol_base_component_t *om_component; +} opened_component_t; + +/* + * Function for selecting one component from all those that are + * available. + * + * Call the init function on all available components and get their + * priorities. Select the component with the highest priority. All + * other components will be closed and unloaded. The selected component + * will have all of its function pointers saved and returned to the + * caller. + */ +int mca_vprotocol_base_select(bool enable_progress_threads, + bool enable_mpi_threads) +{ + int priority = 0, best_priority = -1; + opal_list_item_t *item = NULL; + mca_base_component_list_item_t *cli = NULL; + mca_vprotocol_base_component_t *component = NULL, *best_component = NULL; + mca_vprotocol_base_module_t *module = NULL, *best_module = NULL; + opal_list_t opened; + opened_component_t *om = NULL; + + /* Traverse the list of available components; call their init + functions. */ + OBJ_CONSTRUCT(&opened, opal_list_t); + for(item = opal_list_get_first(&mca_vprotocol_base_components_available); + opal_list_get_end(&mca_vprotocol_base_components_available) != item; + item = opal_list_get_next(item)) + { + cli = (mca_base_component_list_item_t *) item; + component = (mca_vprotocol_base_component_t *) cli->cli_component; + + V_OUTPUT_VERBOSE(500, "vprotocol select: initializing %s component %s", component->pmlm_version.mca_type_name, component->pmlm_version.mca_component_name); + if (NULL == component->pmlm_init) { + V_OUTPUT_VERBOSE(2, "vprotocol select: no init function; ignoring component %s", component->pmlm_version.mca_component_name); + } + else + { + module = component->pmlm_init(&priority, enable_progress_threads, enable_mpi_threads); + if (NULL == module) { + V_OUTPUT_VERBOSE(2, "vprotocol select: init returned failure for component %s", component->pmlm_version.mca_component_name); + } + else + { + V_OUTPUT_VERBOSE(500, "vprotocol select: component %s init returned priority %d", component->pmlm_version.mca_component_name, priority); + if (priority > best_priority) + { + best_priority = priority; + best_component = component; + best_module = module; + } + + om = malloc(sizeof(opened_component_t)); + if (NULL == om) return OMPI_ERR_OUT_OF_RESOURCE; + OBJ_CONSTRUCT(om, opal_list_item_t); + om->om_component = component; + opal_list_append(&opened, (opal_list_item_t*) om); + } + } + } + + /* Finished querying all components. Check for the bozo case. */ + if (NULL == best_component) { + V_OUTPUT_VERBOSE(2, "vprotocol select: no protocol has returned a positive priority, user don't want fault tolerance"); + } + else + { + /* Save the winner */ + mca_vprotocol_component = *best_component; + mca_vprotocol = *best_module; + } + + /* Finalize all non-selected components */ + for (item = opal_list_remove_first(&opened); + NULL != item; + item = opal_list_remove_first(&opened)) + { + om = (opened_component_t *) item; + if (om->om_component != best_component) { + /* Finalize */ + V_OUTPUT_VERBOSE(500, "vprotocol select: component %s not selected / finalized", om->om_component->pmlm_version.mca_component_name); + if (NULL != om->om_component->pmlm_finalize) { + /* Blatently ignore the return code (what would we do to + recover, anyway? This component is going away, so errors + don't matter anymore) */ + om->om_component->pmlm_finalize(); + } + } + OBJ_DESTRUCT(om); + free(om); + } + + mca_base_components_close(pml_v_output, + &mca_vprotocol_base_components_available, + (mca_base_component_t *) best_component); + + /* All done */ + if(best_component != NULL) + { + V_OUTPUT_VERBOSE(500, "vprotocol select: component %s selected", mca_vprotocol_component.pmlm_version.mca_component_name); + return OMPI_SUCCESS; + } + else + return OMPI_ERR_NOT_FOUND; +} diff --git a/ompi/mca/pml/v/pml_v.c b/ompi/mca/pml/v/pml_v.c index 0c3b6bbe95..6215f6c475 100644 --- a/ompi/mca/pml/v/pml_v.c +++ b/ompi/mca/pml/v/pml_v.c @@ -11,8 +11,4 @@ #include "ompi_config.h" #include "pml_v.h" -mca_pml_v_t mca_pml_v = -{ - -1, /* PML_V output */ - {{0,0,0,""}} /* MCA_VPROTOCOL, type is "" to know when no protocol is selected */ -}; +mca_pml_v_t mca_pml_v; diff --git a/ompi/mca/pml/v/pml_v.h b/ompi/mca/pml/v/pml_v.h index 92aa470f49..e50b8cad6d 100644 --- a/ompi/mca/pml/v/pml_v.h +++ b/ompi/mca/pml/v/pml_v.h @@ -14,20 +14,15 @@ #include "ompi_config.h" #include "ompi/mca/pml/pml.h" -#include "pml_v_protocol.h" - #if defined(c_plusplus) || defined(__cplusplus) extern "C" { #endif struct mca_pml_v_t { - int output; - mca_pml_v_protocol_base_component_t protocol_component; - mca_pml_v_protocol_base_module_t protocol; - mca_pml_base_component_t host_pml_component; - mca_pml_base_module_t host_pml; - size_t host_pml_req_recv_size; - size_t host_pml_req_send_size; + mca_pml_base_component_t host_pml_component; + mca_pml_base_module_t host_pml; + size_t host_pml_req_recv_size; + size_t host_pml_req_send_size; }; typedef struct mca_pml_v_t mca_pml_v_t; diff --git a/ompi/mca/pml/v/pml_v_component.c b/ompi/mca/pml/v/pml_v_component.c index 048e5eba3b..a19d0b6317 100644 --- a/ompi/mca/pml/v/pml_v_component.c +++ b/ompi/mca/pml/v/pml_v_component.c @@ -9,20 +9,15 @@ */ #include "ompi_config.h" -#include -#include -#include -#ifdef HAVE_UNISTD -# include -#endif -#include "opal/event/event.h" #include "opal/mca/base/mca_base_param.h" #include "opal/mca/base/base.h" #include "opal/mca/base/mca_base_component_repository.h" +#include "ompi/constants.h" #include "ompi/mca/pml/base/base.h" #include "pml_v.h" -#include "pml_v_protocol_base.h" +#include "vprotocol.h" +#include "base/base.h" static int mca_pml_v_component_open(void); static int mca_pml_v_component_close(void); @@ -32,6 +27,8 @@ static mca_pml_base_module_t *mca_pml_v_component_init(int* priority, bool enabl static int mca_pml_v_component_finalize(void); static int mca_pml_v_component_parasite_finalize(void); +static int mca_pml_v_enable(bool enable); + static inline int mca_pml_v_param_register_int( const char* param_name, int default_value); static inline char *mca_pml_v_param_register_string( const char* param_name, char *default_value); @@ -58,201 +55,187 @@ mca_pml_base_component_1_0_0_t mca_pml_v_component = mca_pml_v_component_finalize /* component finalize */ }; -/******************************************************************************/ -/* MCA level functions */ +static bool pml_v_enable_progress_treads = OMPI_ENABLE_PROGRESS_THREADS; +static bool pml_v_enable_mpi_threads = OMPI_ENABLE_MPI_THREADS; -/* Register MCA parameters and setup verbose output system + + +/******************************************************************************* + * MCA level functions - parasite setup */ static int mca_pml_v_component_open(void) { char *output; int verbose; - + int priority; + + priority = mca_pml_v_param_register_int("priority", -1); output = mca_pml_v_param_register_string("output", "stderr"); verbose = mca_pml_v_param_register_int("verbose", 0); - pml_v_output_init(output, verbose); + pml_v_output_open(output, verbose); + if(-1 != priority) + V_OUTPUT_ERR("pml_v: Overriding priority setting (%d) with -1. The PML V should NEVER be the selected component; even when enabling fault tolerance.", priority); + V_OUTPUT_VERBOSE(500, "loaded"); - return mca_pml_v_protocol_base_load_all(); + return mca_vprotocol_base_open(); } -/* As any parasit, I dont want to die, so I will grab the mca_pml interface - * and put those from the selected vprotocol instead. Once this is done, - * I will do my stuff and call the real func and die only when my host pml - * will. - */ static int mca_pml_v_component_close(void) -{ - /* choose one protocol (ignoring errors as we will be discarded soon if - some occured) */ - /* TODO: dirty trick until pml_base_select gets fixed - have to be moved back to finalize */ - /* TODO: check for bozo case: no selected PML */ - /* mca_pml_v_protocol_base_select(enable_progress_threads, enable_mpi_threads); */ +{ + int ret; + + V_OUTPUT_VERBOSE(500, "component_close: I don't want to be unloaded now."); - - /* Gather some informations about the environment and selects the best protocol - * considering environment and priority */ - mca_pml_v_protocol_base_select(0, 0); - - if(!strcmp(mca_pml_v.protocol_component.pmlm_version.mca_type_name, "vprotocol")) - { - /* ok, we have loaded a fault tolerant protocol, lets go */ - V_OUTPUT_VERBOSE(1, "I don't want to die: I will parasite %s host component %s with %s %s", - mca_pml_base_selected_component.pmlm_version.mca_type_name, - mca_pml_base_selected_component.pmlm_version.mca_component_name, - mca_pml_v.protocol_component.pmlm_version.mca_type_name, - mca_pml_v.protocol_component.pmlm_version.mca_component_name); - - /* setting selected vprotocol mpi functions instead of host's one */ - mca_pml_v.host_pml = mca_pml; /* saving */ - /* protocol stuff */ - if(mca_pml_v.protocol.add_procs) mca_pml.pml_add_procs = mca_pml_v.protocol.add_procs; - if(mca_pml_v.protocol.del_procs) mca_pml.pml_del_procs = mca_pml_v.protocol.del_procs; - if(mca_pml_v.protocol.enable) mca_pml.pml_enable = mca_pml_v.protocol.enable; - if(mca_pml_v.protocol.progress) mca_pml.pml_progress = mca_pml_v.protocol.progress; - if(mca_pml_v.protocol.add_comm) mca_pml.pml_add_comm = mca_pml_v.protocol.add_comm; - if(mca_pml_v.protocol.del_comm) mca_pml.pml_del_comm = mca_pml_v.protocol.del_comm; - if(mca_pml_v.protocol.irecv_init) mca_pml.pml_irecv_init = mca_pml_v.protocol.irecv_init; - if(mca_pml_v.protocol.irecv) mca_pml.pml_irecv = mca_pml_v.protocol.irecv; - if(mca_pml_v.protocol.recv) mca_pml.pml_recv = mca_pml_v.protocol.recv; - if(mca_pml_v.protocol.isend_init) mca_pml.pml_isend_init = mca_pml_v.protocol.isend_init; - if(mca_pml_v.protocol.isend) mca_pml.pml_isend = mca_pml_v.protocol.isend; - if(mca_pml_v.protocol.send) mca_pml.pml_send = mca_pml_v.protocol.send; - if(mca_pml_v.protocol.iprobe) mca_pml.pml_iprobe = mca_pml_v.protocol.iprobe; - if(mca_pml_v.protocol.probe) mca_pml.pml_probe = mca_pml_v.protocol.probe; - if(mca_pml_v.protocol.start) mca_pml.pml_start = mca_pml_v.protocol.start; - if(mca_pml_v.protocol.dump) mca_pml.pml_dump = mca_pml_v.protocol.dump; - - /* Add some extra space for Vprotocol at the end of each PML request - * build a custom request obj class from the original PML request class - * with enough trailing space and rebuild the request pool - */ - if(mca_pml_v.protocol.req_recv_class) - { - ompi_free_list_t pml_fl_save = mca_pml_base_recv_requests; - mca_pml_v.host_pml_req_recv_size = pml_fl_save.fl_elem_class->cls_sizeof; - V_OUTPUT_VERBOSE(300, "req_rebuild: recv\tsize %ld+%ld\talignment=%ld", (long) mca_pml_v.host_pml_req_recv_size, (long) mca_pml_v.protocol.req_recv_class->cls_sizeof, (long) pml_fl_save.fl_alignment); - mca_pml_v.protocol.req_recv_class->cls_parent = pml_fl_save.fl_elem_class; - mca_pml_v.protocol.req_recv_class->cls_sizeof += pml_fl_save.fl_elem_class->cls_sizeof; - /* rebuild the requests free list with the right size */ - OBJ_DESTRUCT(&mca_pml_base_recv_requests); - OBJ_CONSTRUCT(&mca_pml_base_recv_requests, ompi_free_list_t); - ompi_free_list_init_ex(&mca_pml_base_recv_requests, - mca_pml_v.protocol.req_recv_class->cls_sizeof, - pml_fl_save.fl_alignment, - mca_pml_v.protocol.req_recv_class, - pml_fl_save.fl_num_allocated, - pml_fl_save.fl_max_to_alloc, - pml_fl_save.fl_num_per_alloc, - pml_fl_save.fl_mpool, - pml_fl_save.item_init, - pml_fl_save.ctx); - } - if(mca_pml_v.protocol.req_send_class) - { - ompi_free_list_t pml_fl_save = mca_pml_base_send_requests; - mca_pml_v.host_pml_req_send_size = pml_fl_save.fl_elem_class->cls_sizeof; - V_OUTPUT_VERBOSE(300, "req_rebuild: send\tsize %ld+%ld\talignment=%ld", (long) mca_pml_v.host_pml_req_send_size, (long) mca_pml_v.protocol.req_send_class->cls_sizeof, (long) pml_fl_save.fl_alignment); - mca_pml_v.protocol.req_send_class->cls_parent = pml_fl_save.fl_elem_class; - mca_pml_v.protocol.req_send_class->cls_sizeof += pml_fl_save.fl_elem_class->cls_sizeof; - /* rebuild the requests free list with the right size */ - OBJ_DESTRUCT(&mca_pml_base_send_requests); - OBJ_CONSTRUCT(&mca_pml_base_send_requests, ompi_free_list_t); - ompi_free_list_init_ex(&mca_pml_base_send_requests, - mca_pml_v.protocol.req_send_class->cls_sizeof, - pml_fl_save.fl_alignment, - mca_pml_v.protocol.req_send_class, - pml_fl_save.fl_num_allocated, - pml_fl_save.fl_max_to_alloc, - pml_fl_save.fl_num_per_alloc, - pml_fl_save.fl_mpool, - pml_fl_save.item_init, - pml_fl_save.ctx); - } - - /* setting own close and finalize instead of host's one */ + /* Save original PML before making any changes */ mca_pml_v.host_pml_component = mca_pml_base_selected_component; - snprintf(mca_pml_base_selected_component.pmlm_version.mca_component_name, - MCA_BASE_MAX_TYPE_NAME_LEN, "%s]v%s", - mca_pml_v.host_pml_component.pmlm_version.mca_component_name, - mca_pml_v.protocol_component.pmlm_version.mca_component_name); - mca_pml_base_selected_component.pmlm_version.mca_close_component = mca_pml_v_component_parasite_close; - mca_pml_base_selected_component.pmlm_finalize = mca_pml_v_component_parasite_finalize; -#if 0 - /* may be useless ? If not have to parse the entire PML available list to find currently selected */ - component->pmlm_version.mca_close_component = mca_pml_v_component_parasite_close; - component->pmlm_finalize = mca_pml_v_component_parasite_finalize; -#endif + mca_pml_v.host_pml = mca_pml; - V_OUTPUT_VERBOSE(500, "close: I don't want to be unloaded. Referencing myself as using myself"); - if(OPAL_SUCCESS != mca_base_component_repository_retain_component("pml", "v")) + ret = mca_base_component_repository_retain_component("pml", "v"); + if(OPAL_SUCCESS != ret) { - opal_output(0, "close: can't retain myself !"); - return OMPI_ERROR; + V_OUTPUT_ERR("pml_v: component_close: can't retain myself !"); + return ret; } + + /* Mark that we have changed something */ + snprintf(mca_pml_base_selected_component.pmlm_version.mca_component_name, + MCA_BASE_MAX_TYPE_NAME_LEN, "%s]v%s", + mca_pml_v.host_pml_component.pmlm_version.mca_component_name, + mca_vprotocol_component.pmlm_version.mca_component_name); + + /* Replace finalize */ + mca_pml_base_selected_component.pmlm_finalize = + mca_pml_v_component_parasite_finalize; + + /* Make sure we get initialized if some Vprotocol is enabled */ + mca_pml.pml_enable = mca_pml_v_enable; + return OMPI_SUCCESS; - } - V_OUTPUT_VERBOSE(1, "No fault tolerant protocol selected, I let them kill me"); - return OMPI_SUCCESS; } -/* MCA replacement close for host parasited pml component */ +/******************************************************************************* + * Parasite cleanup + */ +static int mca_pml_v_component_parasite_finalize(void) +{ + mca_base_component_list_item_t *cli = NULL; + + V_OUTPUT_VERBOSE(500, "parasite_finalize"); + + /* Make sure we'll get closed again with the true close function */ + mca_pml_v_component.pmlm_version.mca_close_component = + mca_pml_v_component_parasite_close; + cli = OBJ_NEW(mca_base_component_list_item_t); + cli->cli_component = (mca_base_component_t *) &mca_pml_v_component; + opal_list_prepend(&mca_pml_base_components_available, + (opal_list_item_t *) cli); + + /* finalize vprotocol component */ + if(mca_vprotocol_base_selected()) + mca_vprotocol_component.pmlm_finalize(); + + if(mca_pml_v.host_pml_component.pmlm_finalize != NULL) + return mca_pml_v.host_pml_component.pmlm_finalize(); + else + return OMPI_SUCCESS; +} + static int mca_pml_v_component_parasite_close(void) { - V_OUTPUT_VERBOSE(1, "parasite_close: Ok, host component %s is closing, so I accept to die", + V_OUTPUT_VERBOSE(500, "parasite_close: Ok, host component %s is closing, so I accept to die", mca_pml_v.host_pml_component.pmlm_version.mca_component_name); - mca_pml = mca_pml_v.host_pml; mca_pml_base_selected_component = mca_pml_v.host_pml_component; -/* TODO: close the vprotocol component opened in open */ - -/* TODO: I have to dlclose myself, but if I do it from my own code it crashes. - waiting for somebody else to give me tools to do a cleaner stuff - for now, the closed component stays in memory - if(dlclose(myself_dlhandler)) opal_output(mca_pml_v_output, "pml_v.parasite_close: dlclose failed %s", dlerror()); -*/ + mca_vprotocol_base_close(); + pml_v_output_close(); - pml_v_output_finalize(); - return mca_pml_v.host_pml_component.pmlm_version.mca_close_component(); + mca_pml.pml_enable = mca_pml_v.host_pml.pml_enable; + /* don't need to call the host component's close: pml_base will do it */ + return OMPI_SUCCESS; /* ignore any errors as we are leaving anyway */ } -/******************************************************************************/ -/* MCA_PML level functions */ +/******************************************************************************* + * Init/finalize for MCA PML components + */ static mca_pml_base_module_t *mca_pml_v_component_init(int *priority, bool enable_progress_threads, bool enable_mpi_threads) { - V_OUTPUT_VERBOSE(10, "init: I'm not supposed to be here until BTL loading stuff gets fixed!? That's bad..."); - /* I NEVER want to be the selected PML, so I report less than possible - * priority and a NULL module */ - *priority = -1; - return NULL; + V_OUTPUT_VERBOSE(1, "init: I'm not supposed to be here until BTL loading stuff gets fixed!? That's strange..."); + + pml_v_enable_progress_treads = enable_progress_threads; + pml_v_enable_mpi_threads = enable_mpi_threads; + + /* I NEVER want to be the selected PML, so I report less than possible + * priority and a NULL module + */ + *priority = -1; + return NULL; } static int mca_pml_v_component_finalize(void) { - V_OUTPUT_VERBOSE(10, "finalize: I'm not supposed to be here until BTL loading stuff gets fixed!? That's bad..."); - return OMPI_SUCCESS; -} - -static int mca_pml_v_component_parasite_finalize(void) -{ - V_OUTPUT_VERBOSE(100, "parasite_finalize"); - - /* finalize vprotocol component */ - mca_pml_v.protocol_component.pmlm_finalize(); - - if(mca_pml_v.host_pml_component.pmlm_finalize != NULL) - return mca_pml_v.host_pml_component.pmlm_finalize(); - else + V_OUTPUT_VERBOSE(1, "finalize: I'm not supposed to be here until BTL loading stuff gets fixed!? That's strange..."); + /* Nothing to do here. We are not sure we need to be unloaded or not at + * this stage + */ return OMPI_SUCCESS; } -/******************************************************************************/ -/* utilities */ +/******************************************************************************* + * Enable the PML V (and initialize the Vprotocol) + */ +static int mca_pml_v_enable(bool enable) +{ + int ret; + + /* Enable the real PML (no threading issues there as threads are started + * later + */ + ret = mca_pml_v.host_pml.pml_enable(enable); + if(OMPI_SUCCESS != ret) return ret; + + if(enable) { + if(mca_vprotocol_base_selected()) { + if(mca_vprotocol.enable) + return mca_vprotocol.enable(enable); + else + return OMPI_SUCCESS; + } + + mca_vprotocol_base_select(pml_v_enable_progress_treads, + pml_v_enable_mpi_threads); + + if(mca_vprotocol_base_selected()) { + V_OUTPUT_VERBOSE(1, "I don't want to die: I will parasite %s host component %s with %s %s", + mca_pml_base_selected_component.pmlm_version.mca_type_name, + mca_pml_base_selected_component.pmlm_version.mca_component_name, + mca_vprotocol_component.pmlm_version.mca_type_name, + mca_vprotocol_component.pmlm_version.mca_component_name); + + ret = mca_vprotocol_base_parasite(); + if(OMPI_SUCCESS != ret) return ret; + if(mca_vprotocol.enable) + return mca_vprotocol.enable(enable); + else + return OMPI_SUCCESS; + } + V_OUTPUT_VERBOSE(1, "No fault tolerant protocol selected. All are unloaded"); + } + /* Disable */ + mca_pml = mca_pml_v.host_pml; + mca_pml.pml_enable = mca_pml_v_enable; + return OMPI_SUCCESS; +} + + +/******************************************************************************* + * utilities + */ static inline int mca_pml_v_param_register_int( const char* param_name, int default_value ) { diff --git a/ompi/mca/pml/v/pml_v_output.c b/ompi/mca/pml/v/pml_v_output.c index d4d3b5eb7b..a9dd665bd4 100644 --- a/ompi/mca/pml/v/pml_v_output.c +++ b/ompi/mca/pml/v/pml_v_output.c @@ -10,13 +10,19 @@ #include "pml_v_output.h" +#if defined(HAVE_UNISTD_H) +#include +#endif + +int pml_v_output = -1; + int pml_v_output_open(char *output, int verbosity) { opal_output_stream_t lds; char hostname[32] = "NA"; OBJ_CONSTRUCT(&lds, opal_output_stream_t); if(!output) { - mca_pml_v.output = 0; + pml_v_output = 0; } else { if(!strcmp(output, "stdout")) { @@ -34,12 +40,12 @@ int pml_v_output_open(char *output, int verbosity) { gethostname(hostname, 32); asprintf(&lds.lds_prefix, "[%s:%05d] pml_v: ", hostname, getpid()); lds.lds_verbose_level = verbosity; - mca_pml_v.output = opal_output_open(&lds); + pml_v_output = opal_output_open(&lds); free(lds.lds_prefix); } - return mca_pml_v.output; + return pml_v_output; } void pml_v_output_close(void) { - opal_output_close(mca_pml_v.output); + opal_output_close(pml_v_output); } diff --git a/ompi/mca/pml/v/pml_v_output.h b/ompi/mca/pml/v/pml_v_output.h index 27d36c47d9..f44a6cf3de 100644 --- a/ompi/mca/pml/v/pml_v_output.h +++ b/ompi/mca/pml/v/pml_v_output.h @@ -12,14 +12,15 @@ #define PML_V_OUTPUT_H_HAS_BEEN_INCLUDED #include "ompi_config.h" -#include "pml_v.h" - -#include +#include "opal/util/output.h" +#include #if defined(c_plusplus) || defined(__cplusplus) extern "C" { #endif +extern int pml_v_output; + int pml_v_output_open(char *output, int verbosity); void pml_v_output_close(void); @@ -40,16 +41,16 @@ static inline void V_OUTPUT_ERR(const char *fmt, ... ) /* Tricky stuff to define V_OUTPUT and V_OUTPUT_VERBOSE with variadic arguments */ #if defined(ACCEPT_C99) -# define V_OUTPUT(ARGS...) \ - OPAL_OUTPUT((mca_pml_v.output, __VA_ARGS__)) -# define V_OUTPUT_VERBOSE(V, ARGS...) \ - OPAL_OUTPUT_VERBOSE((V, mca_pml_v.output, __VA_ARGS__)) +# define V_OUTPUT(ARGS...) \ + OPAL_OUTPUT((pml_v_output, __VA_ARGS__)) +# define V_OUTPUT_VERBOSE(V, ARGS...) \ + OPAL_OUTPUT_VERBOSE((V, pml_v_output, __VA_ARGS__)) #elif defined(__GNUC__) && !defined(__STDC__) -# define V_OUTPUT(ARGS...) \ - OPAL_OUTPUT((mca_pml_v.output, ARGS)) -# define V_OUTPUT_VERBOSE(V, ARGS...) \ - OPAL_OUTPUT_VERBOSE((V, mca_pml_v.output, ARGS)) +# define V_OUTPUT(ARGS...) \ + OPAL_OUTPUT((pml_v_output, ARGS)) +# define V_OUTPUT_VERBOSE(V, ARGS...) \ + OPAL_OUTPUT_VERBOSE((V, pml_v_output, ARGS)) #elif OMPI_ENABLE_DEBUG /* No variadic macros available... So sad */ @@ -62,7 +63,7 @@ static inline void V_OUTPUT(const char* fmt, ... ) va_start(list, fmt); ret = vasprintf(&str, fmt, list); assert(-1 != ret); - opal_output(mca_pml_v.output, str); + opal_output(pml_v_output, str); free(str); va_end(list); } @@ -74,7 +75,7 @@ static inline void V_OUTPUT_VERBOSE(int V, const char* fmt, ... ) { va_start(list, fmt); ret = vasprintf(&str, fmt, list); assert(-1 != ret); - opal_output_verbose(V, mca_pml_v.output, str); + opal_output_verbose(V, pml_v_output, str); free(str); va_end(list); } diff --git a/ompi/mca/pml/v/pml_v_protocol.h b/ompi/mca/pml/v/pml_v_protocol.h deleted file mode 100644 index 5034fc8fb3..0000000000 --- a/ompi/mca/pml/v/pml_v_protocol.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef __INCLUDE_V_PROTOCOL_H_ -#define __INCLUDE_V_PROTOCOL_H_ - -#include "ompi_config.h" -#include "opal/mca/mca.h" -#include "ompi/request/request.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/btl/btl.h" -#include "ompi/mca/pml/base/pml_base_recvreq.h" -#include "ompi/mca/pml/base/pml_base_sendreq.h" - -#if defined(c_plusplus) || defined(__cplusplus) -extern "C" { -#endif - -/** - * PML_V->PROTOCOL Called by MCA_PML_V framework to initialize the component. - * - * @param priority (OUT) Relative priority or ranking used by MCA to - * select a component. - * - * @param enable_progress_threads (IN) Whether this component is - * allowed to run a hidden/progress thread or not. - * - * @param enable_mpi_threads (IN) Whether support for multiple MPI - * threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which - * indicates whether multiple threads may invoke this component - * simultaneously or not. - */ -typedef struct mca_pml_v_protocol_base_module_1_0_0_t * (*mca_pml_v_protocol_base_component_init_fn_t)( - int *priority, - bool enable_progress_threads, - bool enable_mpi_threads); -typedef int (*mca_pml_v_protocol_base_component_finalize_fn_t)(void); - -/** - * PML_V->PROTOCOL Called by VPROTOCOL to get the actual address of the - * protocol specific part of the request - * - * @param request (IN) the address of an ompi_request - * @return address of the mca_vprotocol_base_request_t associated with the request - */ -#define VPROTOCOL_RECV_REQ(req) \ - (((char *) req) + mca_pml_v.host_pml_req_recv_size) -#define VPROTOCOL_SEND_REQ(req) \ - (((char *) req) + mca_pml_v.host_pml_req_send_size) -#if OMPI_ENABLE_DEBUG -#define VPROTOCOL_REQ(req) ( \ - (MCA_PML_REQUEST_SEND == ((mca_pml_base_request_t *)req)->req_type) \ - ? VPROTOCOL_SEND_REQ(req) \ - : ( \ - (MCA_PML_REQUEST_RECV == ((mca_pml_base_request_t *)req)->req_type) \ - ? VPROTOCOL_RECV_REQ(req) \ - : (V_OUTPUT("VPROTOCOL_REQ: error, request type is %d (expected %d or %d)", ((mca_pml_base_request_t *)req)->req_type, MCA_PML_REQUEST_SEND, MCA_PML_REQUEST_RECV),NULL) \ - ) \ -) -#else -#define VPROTOCOL_REQ(req) \ - ((((mca_pml_base_request_t *) req)->req_type == MCA_PML_REQUEST_SEND) \ - ? VPROTOCOL_SEND_REQ(req) \ - : VPROTOCOL_RECV_REQ(req)) -#endif - -/** - * PML_V_PROTOCOL component version and interface functions. - */ -typedef struct mca_pml_v_protocol_base_1_0_0_t -{ - mca_base_component_t pmlm_version; - mca_base_component_data_1_0_0_t pmlm_data; - mca_pml_v_protocol_base_component_init_fn_t pmlm_init; - mca_pml_v_protocol_base_component_finalize_fn_t pmlm_finalize; -} mca_pml_v_protocol_base_component_1_0_0_t; -typedef mca_pml_v_protocol_base_component_1_0_0_t mca_pml_v_protocol_base_component_t; - - -typedef struct mca_pml_v_protocol_base_module_1_0_0_t -{ - /* PML module stuff */ - mca_pml_base_module_add_procs_fn_t add_procs; - mca_pml_base_module_del_procs_fn_t del_procs; - mca_pml_base_module_enable_fn_t enable; - mca_pml_base_module_progress_fn_t progress; - mca_pml_base_module_add_comm_fn_t add_comm; - mca_pml_base_module_del_comm_fn_t del_comm; - mca_pml_base_module_irecv_init_fn_t irecv_init; - mca_pml_base_module_irecv_fn_t irecv; - mca_pml_base_module_recv_fn_t recv; - mca_pml_base_module_isend_init_fn_t isend_init; - mca_pml_base_module_isend_fn_t isend; - mca_pml_base_module_send_fn_t send; - mca_pml_base_module_iprobe_fn_t iprobe; - mca_pml_base_module_probe_fn_t probe; - mca_pml_base_module_start_fn_t start; - mca_pml_base_module_dump_fn_t dump; - - /* Request extra data */ - opal_class_t * req_recv_class; - opal_class_t * req_send_class; - } mca_pml_v_protocol_base_module_1_0_0_t; -typedef mca_pml_v_protocol_base_module_1_0_0_t mca_pml_v_protocol_base_module_t; - -#if defined(c_plusplus) || defined(__cplusplus) -} -#endif - -#endif /* __INCLUDE_V_PROTOCOL_H_ */ diff --git a/ompi/mca/pml/v/pml_v_protocol_base.c b/ompi/mca/pml/v/pml_v_protocol_base.c deleted file mode 100644 index 954cc1b5cd..0000000000 --- a/ompi/mca/pml/v/pml_v_protocol_base.c +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "opal/event/event.h" -#include "opal/mca/mca.h" -#include "opal/mca/base/mca_base_param.h" -#include "pml_v_protocol.h" -#include "pml_v_protocol_base.h" -#include "pml_v.h" -#include "static-components.h" - -static opal_list_t mca_pml_v_protocol_base_components_available; - -/** - * Load any vprotocol MCA component, stored in available_vprotocol_components - * call open function of all those components. - */ -int mca_pml_v_protocol_base_load_all(void) -{ - OBJ_CONSTRUCT(&mca_pml_v_protocol_base_components_available, opal_list_t); - return mca_base_components_open("vprotocol", 0, mca_pml_v_protocol_base_static_components, - &mca_pml_v_protocol_base_components_available, true); -} - -typedef struct opened_component_t { - opal_list_item_t super; - mca_pml_v_protocol_base_component_t *om_component; -} opened_component_t; - -/** - * Function for selecting one component from all those that are - * available. - * - * Call the init function on all available components and get their - * priorities. Select the component with the highest priority. All - * other components will be closed and unloaded. The selected component - * will have all of its function pointers saved and returned to the - * caller. - */ -int mca_pml_v_protocol_base_select(bool enable_progress_threads, bool enable_mpi_threads) -{ - int priority = 0, best_priority = -1; - opal_list_item_t *item = NULL; - mca_base_component_list_item_t *cli = NULL; - mca_pml_v_protocol_base_component_t *component = NULL, *best_component = NULL; - mca_pml_v_protocol_base_module_t *module = NULL, *best_module = NULL; - opal_list_t opened; - opened_component_t *om = NULL; - - /* Traverse the list of available components; call their init - functions. */ - OBJ_CONSTRUCT(&opened, opal_list_t); - for(item = opal_list_get_first(&mca_pml_v_protocol_base_components_available); - opal_list_get_end(&mca_pml_v_protocol_base_components_available) != item; - item = opal_list_get_next(item)) { - cli = (mca_base_component_list_item_t *) item; - component = (mca_pml_v_protocol_base_component_t *) cli->cli_component; - - V_OUTPUT_VERBOSE(500, "vprotocol select: initializing %s component %s", component->pmlm_version.mca_type_name, component->pmlm_version.mca_component_name); - if (NULL == component->pmlm_init) { - V_OUTPUT_VERBOSE(2, "vprotocol select: no init function; ignoring component %s", component->pmlm_version.mca_component_name); - } - else - { - module = component->pmlm_init(&priority, enable_progress_threads, enable_mpi_threads); - if (NULL == module) { - V_OUTPUT_VERBOSE(2, "vprotocol select: init returned failure for component %s", component->pmlm_version.mca_component_name); - } - else - { - V_OUTPUT_VERBOSE(500, "vprotocol select: component %s init returned priority %d", component->pmlm_version.mca_component_name, priority); - if (priority > best_priority) - { - best_priority = priority; - best_component = component; - best_module = module; - } - - om = malloc(sizeof(opened_component_t)); - if (NULL == om) return OMPI_ERR_OUT_OF_RESOURCE; - OBJ_CONSTRUCT(om, opal_list_item_t); - om->om_component = component; - opal_list_append(&opened, (opal_list_item_t*) om); - } - } - } - - /* Finished querying all components. Check for the bozo case. */ - if (NULL == best_component) { - V_OUTPUT_VERBOSE(2, "vprotocol select: no protocol has returned a positive priority, user don't want fault tolerance"); - } - else - { - /* Save the winner */ - mca_pml_v.protocol_component = *best_component; - mca_pml_v.protocol = *best_module; - } - - /* Finalize all non-selected components */ - for (item = opal_list_remove_first(&opened); - NULL != item; - item = opal_list_remove_first(&opened)) { - om = (opened_component_t *) item; - if (om->om_component != best_component) { - /* Finalize */ - V_OUTPUT_VERBOSE(500, "vprotocol select: component %s not selected / finalized", om->om_component->pmlm_version.mca_component_name); - if (NULL != om->om_component->pmlm_finalize) { - /* Blatently ignore the return code (what would we do to - recover, anyway? This component is going away, so errors - don't matter anymore) */ - om->om_component->pmlm_finalize(); - } - } - OBJ_DESTRUCT(om); - free(om); - } - - /* This base function closes, unloads, and removes from the - available list all unselected components. The available list will - contain only the selected component (if one). */ - mca_base_components_close(mca_pml_v.output, - &mca_pml_v_protocol_base_components_available, - (mca_base_component_t *) best_component); - - - /* All done */ - if(best_component != NULL) - { - V_OUTPUT_VERBOSE(500, "vprotocol select: component %s selected", mca_pml_v.protocol_component.pmlm_version.mca_component_name); - return OMPI_SUCCESS; - } - else - return OMPI_ERR_NOT_FOUND; -} diff --git a/ompi/mca/pml/v/pml_v_protocol_base.h b/ompi/mca/pml/v/pml_v_protocol_base.h deleted file mode 100644 index c8b14c5224..0000000000 --- a/ompi/mca/pml/v/pml_v_protocol_base.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef __INCLUDE_V_PROTOCOL_BASE_H_ -#define __INCLUDE_V_PROTOCOL_BASE_H_ - -#include "ompi_config.h" -#include "ompi/request/request.h" -#include "opal/mca/mca.h" -#include "ompi/mca/pml/pml.h" -#include "pml_v_protocol.h" - -#if defined(c_plusplus) || defined(__cplusplus) -extern "C" { -#endif - -OMPI_DECLSPEC int mca_pml_v_protocol_base_load_all(void); -OMPI_DECLSPEC int mca_pml_v_protocol_base_select(bool enable_progress_threads, bool enable_mpi_threads); - -/* - * Macro for use in components that are of type vprotocol v1.0.0 - */ -#define MCA_VPROTOCOL_BASE_VERSION_1_0_0 \ - /* vprotocol v1.0 is chained to MCA v1.0 */ \ - MCA_BASE_VERSION_1_0_0, \ - /* vprotocol v1.0 */ \ - "vprotocol", 1, 0, 0 - -#if defined(c_plusplus) || defined(__cplusplus) -} -#endif - -#endif /* __INCLUDE_V_PROTOCOL_BASE_H_ */ diff --git a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist.h b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist.h index 322ce1d3b2..3ddeaa6faf 100644 --- a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist.h +++ b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist.h @@ -8,18 +8,18 @@ * $HEADER$ */ +#include "vprotocol_pessimist_sender_based.h" + #ifndef __INCLUDE_VPROTOCOL_PESSIMIST_H__ #define __INCLUDE_VPROTOCOL_PESSIMIST_H__ #include "ompi_config.h" -#include "../pml_v.h" -#include "../pml_v_protocol.h" +#include "../vprotocol.h" #include "ompi/communicator/communicator.h" #include "vprotocol_pessimist_event.h" -#include "vprotocol_pessimist_sender_based.h" typedef struct mca_vprotocol_pessimist_module_t { - mca_pml_v_protocol_base_module_t super; + mca_vprotocol_base_module_t super; /* event tagging clock */ vprotocol_pessimist_clock_t clock; diff --git a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_component.c b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_component.c index 488c31028e..d7c8feed56 100644 --- a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_component.c +++ b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_component.c @@ -12,18 +12,15 @@ #include "opal/mca/mca.h" #include "opal/mca/base/mca_base_param.h" -#include "../pml_v.h" -#include "../pml_v_protocol_base.h" #include "vprotocol_pessimist.h" - static inline int mca_param_register_int( const char* param_name, int default_value); static inline char *mca_param_register_string(const char* param_name, char *default_value); static int mca_vprotocol_pessimist_component_open(void); static int mca_vprotocol_pessimist_component_close(void); -static mca_pml_v_protocol_base_module_t *mca_vprotocol_pessimist_component_init( int* priority, bool, bool); +static mca_vprotocol_base_module_t *mca_vprotocol_pessimist_component_init( int* priority, bool, bool); static int mca_vprotocol_pessimist_component_finalize(void); static int _priority; @@ -34,12 +31,12 @@ static int _sender_based_size; static int _event_buffer_size; static char *_mmap_file_name; -mca_pml_v_protocol_base_component_1_0_0_t mca_vprotocol_pessimist_component = +mca_vprotocol_base_component_1_0_0_t mca_vprotocol_pessimist_component = { /* First, the mca_base_component_t struct containing meta * information about the component itself */ { - /* Indicate that we are a pml v1.0.0 component (which also implies + /* Indicate that we are a vprotocol v1.0.0 component (which also implies a specific MCA version) */ MCA_VPROTOCOL_BASE_VERSION_1_0_0, "pessimist", /* MCA component name */ @@ -71,19 +68,19 @@ static int mca_vprotocol_pessimist_component_open(void) _sender_based_size = mca_param_register_int("sender_based_chunk", 100 * 1024 * 1024); _event_buffer_size = mca_param_register_int("event_buffer_size", 1024); _mmap_file_name = mca_param_register_string("sender_based_file", "vprotocol_pessimist-senderbased"); - V_OUTPUT_VERBOSE(500, "vprotocol_pessimist: open: read priority %d", _priority); + V_OUTPUT_VERBOSE(500, "vprotocol_pessimist: component_open: read priority %d", _priority); return OMPI_SUCCESS; } static int mca_vprotocol_pessimist_component_close(void) { - V_OUTPUT_VERBOSE(500, "vprotocol_pessimist: close"); + V_OUTPUT_VERBOSE(500, "vprotocol_pessimist: component_close"); return OMPI_SUCCESS; } /** VPROTOCOL level functions (same as PML one) */ -static mca_pml_v_protocol_base_module_t *mca_vprotocol_pessimist_component_init( int* priority, +static mca_vprotocol_base_module_t *mca_vprotocol_pessimist_component_init( int* priority, bool enable_progress_threads, bool enable_mpi_threads) { diff --git a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_request.h b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_request.h index 7b5037e8c2..183b31c50d 100644 --- a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_request.h +++ b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_request.h @@ -14,6 +14,7 @@ #include "ompi_config.h" #include "vprotocol_pessimist_event.h" #include "ompi/request/request.h" +#include "../base/vprotocol_base_request.h" #if defined(c_plusplus) || defined(__cplusplus) extern "C" { diff --git a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.c b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.c index 38e59498a9..1c00f304ec 100644 --- a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.c +++ b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.c @@ -10,8 +10,12 @@ #include "vprotocol_pessimist_sender_based.h" #include +#if defined(HAVE_SYS_MMAN_H) #include +#endif /* defined(HAVE_SYS_MMAN_H) */ +#if defined(HAVE_UNISTD_H) #include +#endif #include #define sb mca_vprotocol_pessimist.sender_based @@ -22,7 +26,7 @@ int vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size) sb.sb_offset = 0; sb.sb_length = size; sb.sb_pagesize = getpagesize(); - sb.sb_cursor = sb.sb_addr = NULL; + sb.sb_cursor = sb.sb_addr = (uintptr_t) NULL; sb.sb_vacant = 0; sb.sb_comm = MPI_COMM_NULL; @@ -31,8 +35,8 @@ int vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size) sb.sb_fd = open(path, O_CREAT | O_TRUNC | O_RDWR, 0600); if(-1 == sb.sb_fd) { - opal_output(0, "pml_v: vprotocol_pessimist: sender_based_init: open (%s): %s", - path, strerror(errno)); + V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_init: open (%s): %s", + path, strerror(errno)); return -1; } return sb.sb_fd; @@ -49,15 +53,15 @@ void vprotocol_pessimist_sender_based_finalize(void) * if(MPI_SUCCESS != ret) * opal_output(0, "pml_v: protocol_pessimist: sender_based_finalize: ompi_comm_free failed (%d)", ret); */ - ret = munmap(sb.sb_addr, sb.sb_length); + ret = munmap((void *) sb.sb_addr, sb.sb_length); if(-1 == ret) - opal_output(0, "pml_v: protocol_pessimsit: sender_based_finalize: munmap (%p): %s", - sb.sb_addr, strerror(errno)); + V_OUTPUT_ERR("pml_v: protocol_pessimsit: sender_based_finalize: munmap (%p): %s", + (void *) sb.sb_addr, strerror(errno)); } ret = close(sb.sb_fd); if(-1 == ret) - opal_output(0, "pml_v: protocol_pessimist: sender_based_finalize: close (%d): %s", - sb.sb_fd, strerror(errno)); + V_OUTPUT_ERR("pml_v: protocol_pessimist: sender_based_finalize: close (%d): %s", + sb.sb_fd, strerror(errno)); } @@ -69,43 +73,45 @@ void vprotocol_pessimist_sender_based_alloc(size_t len) if(sb.sb_comm == MPI_COMM_NULL) ompi_comm_dup(MPI_COMM_SELF, &sb.sb_comm, 1); else - munmap(sb.sb_addr, sb.sb_length); + munmap((void *) sb.sb_addr, sb.sb_length); /* Take care of alignement of sb_offset */ - sb.sb_offset += (uintptr_t) sb.sb_cursor - (uintptr_t) sb.sb_addr; - sb.sb_cursor = (char *) ((uintptr_t) (sb.sb_offset % sb.sb_pagesize)); - sb.sb_offset -= (uintptr_t) sb.sb_cursor; + sb.sb_offset += sb.sb_cursor - sb.sb_addr; + sb.sb_cursor = sb.sb_offset % sb.sb_pagesize; + sb.sb_offset -= sb.sb_cursor; /* Adjusting sb_length for the largest application message to fit */ if(sb.sb_length < len) - sb.sb_length = len + (size_t) sb.sb_cursor; + sb.sb_length = len + sb.sb_cursor; if(-1 == lseek(sb.sb_fd, sb.sb_offset + sb.sb_length, SEEK_SET)) { - opal_output(1, "pml_v: vprotocol_pessimist: sender_based_alloc: lseek: %s", - strerror(errno)); + V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: lseek: %s", + strerror(errno)); close(sb.sb_fd); ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE, false); } if(1 != write(sb.sb_fd, "", 1)) { - opal_output(1, "pml_v: vprotocol_pessimist: sender_based_alloc: write: %s", - strerror(errno)); + V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: write: %s", + strerror(errno)); close(sb.sb_fd); ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE, false); } - sb.sb_addr = mmap(sb.sb_addr, sb.sb_length, PROT_WRITE | PROT_READ, - MAP_SHARED, sb.sb_fd, sb.sb_offset); - if((void *) -1 == sb.sb_addr) + sb.sb_addr = (uintptr_t) mmap((void *) sb.sb_addr, sb.sb_length, + PROT_WRITE | PROT_READ, MAP_SHARED, sb.sb_fd, + sb.sb_offset); + if(((uintptr_t) -1) == sb.sb_addr) { - opal_output(1, "pml_v: vprotocol_pessimist: sender_based_alloc: mmap: %s", - strerror(errno)); - close(sb.sb_fd); - ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE, false); + V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: mmap: %s", + strerror(errno)); + close(sb.sb_fd); + ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE, false); } - sb.sb_cursor += (intptr_t) sb.sb_addr; /* absolute addr of sender_based buffer */ - sb.sb_vacant = sb.sb_length - sizeof(vprotocol_pessimist_sender_based_header_t); - V_OUTPUT_VERBOSE(30, "pessimist:\tsb\tgrow\toffset %llu\tlength %llu\tbase %p\tcursor %p", (long long) sb.sb_offset, (long long) sb.sb_length, sb.sb_addr, sb.sb_cursor); + sb.sb_vacant = sb.sb_length - sb.sb_cursor - /* make room for the header */ + sizeof(vprotocol_pessimist_sender_based_header_t); + sb.sb_cursor += sb.sb_addr; /* set absolute addr of sender_based buffer */ + V_OUTPUT_VERBOSE(30, "pessimist:\tsb\tgrow\toffset %llu\tlength %llu\tbase %p\tcursor %p", (unsigned long long) sb.sb_offset, (unsigned long long) sb.sb_length, (void *) sb.sb_addr, (void *) sb.sb_cursor); } #undef sb diff --git a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.h b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.h index 6456e37ca5..194dc24a1a 100644 --- a/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.h +++ b/ompi/mca/pml/v/protocol_pessimist/vprotocol_pessimist_sender_based.h @@ -8,31 +8,29 @@ * $HEADER$ */ -#include "ompi_config.h" -#include "vprotocol_pessimist.h" - #ifndef __VPROTOCOL_PESSIMIST_SENDERBASED_H__ #define __VPROTOCOL_PESSIMIST_SENDERBASED_H__ #include "vprotocol_pessimist_request.h" -#if defined(HAVE_SYS_MMAN_H) -#include -#endif /* defined(HAVE_SYS_MMAN_H) */ +#include "ompi/mca/pml/base/pml_base_sendreq.h" + typedef struct vprotocol_pessimist_sender_based_t { - int sb_pagesize; /* size of memory pages on this architecture */ + int sb_pagesize; /* size of memory pages on this architecture */ ompi_communicator_t *sb_comm; - int sb_fd; /* file descriptor of mapped file */ - off_t sb_offset; /* offset in mmaped file */ + int sb_fd; /* file descriptor of mapped file */ + off_t sb_offset; /* offset in mmaped file */ - char *sb_addr; /* base address of mmaped segment */ - size_t sb_length; /* length of mmaped segment */ - char *sb_cursor; /* current pointer to writeable memory */ - size_t sb_vacant; /* available space before end of segment */ + uintptr_t sb_addr; /* base address of mmaped segment */ + size_t sb_length; /* length of mmaped segment */ + uintptr_t sb_cursor; /* current pointer to writeable memory */ + size_t sb_vacant; /* available space before end of segment */ } vprotocol_pessimist_sender_based_t; +#include "vprotocol_pessimist.h" + typedef struct vprotocol_pessimist_sender_based_header_t { size_t size; @@ -52,7 +50,7 @@ void vprotocol_pessimist_sender_based_finalize(void); void vprotocol_pessimist_sender_based_alloc(size_t len); -#define __SENDER_BASED_IOV_PACK(req) do { \ +#define __SENDER_BASED_CNVTOR_PACK(req) do { \ if( 0 != req->req_bytes_packed ) { \ ompi_convertor_t conv; \ size_t max_data; \ @@ -69,7 +67,7 @@ void vprotocol_pessimist_sender_based_alloc(size_t len); } \ } while(0) -#define __SENDER_BASED_SENDRECV_PACK(req) do { \ +#define __SENDER_BASED_SNDRCV_PACK(req) do { \ mca_pml_v.host_pml.pml_irecv( \ mca_vprotocol_pessimist.sender_based.sb_cursor, \ req->req_bytes_packed, MPI_PACKED, 0, 0, \ @@ -82,6 +80,7 @@ void vprotocol_pessimist_sender_based_alloc(size_t len); &VPESSIMIST_SEND_REQ(req)->sb_reqs[1]); \ } while(0); +#if 0 #define __SENDER_BASED_PACK(req) do { \ vprotocol_pessimist_sender_based_header_t *sbhdr = \ (vprotocol_pessimist_sender_based_header_t *) \ @@ -94,12 +93,33 @@ void vprotocol_pessimist_sender_based_alloc(size_t len); mca_vprotocol_pessimist.sender_based.sb_cursor += \ sizeof(vprotocol_pessimist_sender_based_header_t); \ \ - __SENDER_BASED_SENDRECV_PACK(req); \ + __SENDER_BASED_CNVTOR_PACK(req); \ mca_vprotocol_pessimist.sender_based.sb_cursor += sbhdr->size; \ mca_vprotocol_pessimist.sender_based.sb_vacant -= (sbhdr->size + \ sizeof(vprotocol_pessimist_sender_based_header_t)); \ - V_OUTPUT_VERBOSE(70, "pessimist:\tsb\twrite\t%"PRIpclock"\tsize %lu", VPESSIMIST_REQ(&req->req_base)->reqid, sbhdr->size); \ + V_OUTPUT_VERBOSE(70, "pessimist:\tsb\twrite\t%"PRIpclock"\tsize %lu", VPESSIMIST_REQ(&req->req_base)->reqid, sbhdr->size + sizeof(vprotocol_pessimist_sender_based_header_t); \ } while(0) +#endif + +static inline void __SENDER_BASED_PACK(mca_pml_base_send_request_t *req) { + vprotocol_pessimist_sender_based_header_t *sbhdr = + (vprotocol_pessimist_sender_based_header_t *) + mca_vprotocol_pessimist.sender_based.sb_cursor; + sbhdr->size = req->req_bytes_packed; + sbhdr->dst = req->req_base.req_peer; + sbhdr->tag = req->req_base.req_tag; + sbhdr->contextid = req->req_base.req_comm->c_contextid; + sbhdr->sequence = req->req_base.req_sequence; + mca_vprotocol_pessimist.sender_based.sb_cursor += + sizeof(vprotocol_pessimist_sender_based_header_t); + + __SENDER_BASED_CNVTOR_PACK(req); + mca_vprotocol_pessimist.sender_based.sb_cursor += sbhdr->size; + mca_vprotocol_pessimist.sender_based.sb_vacant -= (sbhdr->size + + sizeof(vprotocol_pessimist_sender_based_header_t)); + V_OUTPUT_VERBOSE(70, "pessimist:\tsb\twrite\t%"PRIpclock"\tsize %lu", VPESSIMIST_REQ(&req->req_base)->reqid, sbhdr->size + sizeof(vprotocol_pessimist_sender_based_header_t)); +} + /** Copy data associated to a pml_base_send_request_t to the sender based * message payload buffer @@ -117,7 +137,11 @@ void vprotocol_pessimist_sender_based_alloc(size_t len); /** Ensure sender based is finished before allowing user to touch send buffer */ -#define VPROTOCOL_PESSIMIST_SENDER_BASED_FLUSH(REQ) do { \ +#define VPROTOCOL_PESSIMIST_SENDER_BASED_FLUSH(REQ) __SENDER_BASED_CNVTOR_FLUSH(REQ) + +#define __SENDER_BASED_CNVTOR_FLUSH(REQ) + +#define __SENDER_BASED_SNDRCV_FLUSH(REQ) do { \ if(NULL != VPESSIMIST_REQ(REQ)->sb_reqs[0]) \ { \ ompi_request_wait_all(2, VPESSIMIST_REQ(REQ)->sb_reqs, \ diff --git a/ompi/mca/pml/v/static-components.h b/ompi/mca/pml/v/static-components.h index 9c36b1b071..8581cab3fb 100644 --- a/ompi/mca/pml/v/static-components.h +++ b/ompi/mca/pml/v/static-components.h @@ -4,7 +4,7 @@ -const mca_base_component_t *mca_pml_v_protocol_base_static_components[] = { +const mca_base_component_t *mca_vprotocol_base_static_components[] = { NULL }; diff --git a/ompi/mca/pml/v/vprotocol.h b/ompi/mca/pml/v/vprotocol.h new file mode 100644 index 0000000000..796830de35 --- /dev/null +++ b/ompi/mca/pml/v/vprotocol.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2004-2007 The Trustees of the University of Tennessee. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef __INCLUDE_VPROTOCOL_H_ +#define __INCLUDE_VPROTOCOL_H_ + +#include "ompi_config.h" +#include "opal/mca/mca.h" +#include "ompi/mca/pml/pml.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/* PML_V->PROTOCOL Called by MCA_PML_V framework to initialize the component. + * + * @param priority (OUT) Relative priority or ranking used by MCA to + * select a component. + * + * @param enable_progress_threads (IN) Whether this component is + * allowed to run a hidden/progress thread or not. + * + * @param enable_mpi_threads (IN) Whether support for multiple MPI + * threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which + * indicates whether multiple threads may invoke this component + * simultaneously or not. + */ +typedef struct mca_vprotocol_base_module_1_0_0_t * + (*mca_vprotocol_base_component_init_fn_t)(int *priority, + bool enable_progress_threads, + bool enable_mpi_threads); + +/* Release any resource allocated in init + */ +typedef int (*mca_vprotocol_base_component_finalize_fn_t)(void); + + +/* The MCA type for class instance + */ +typedef struct mca_vprotocol_base_component_1_0_0_t { + mca_base_component_t pmlm_version; + mca_base_component_data_1_0_0_t pmlm_data; + mca_vprotocol_base_component_init_fn_t pmlm_init; + mca_vprotocol_base_component_finalize_fn_t pmlm_finalize; +} mca_vprotocol_base_component_1_0_0_t; +typedef mca_vprotocol_base_component_1_0_0_t mca_vprotocol_base_component_t; + +/* The base module of the component + */ +typedef struct mca_vprotocol_base_module_1_0_0_t +{ + /* PML module stuff */ + mca_pml_base_module_add_procs_fn_t add_procs; + mca_pml_base_module_del_procs_fn_t del_procs; + mca_pml_base_module_enable_fn_t enable; + mca_pml_base_module_progress_fn_t progress; + mca_pml_base_module_add_comm_fn_t add_comm; + mca_pml_base_module_del_comm_fn_t del_comm; + mca_pml_base_module_irecv_init_fn_t irecv_init; + mca_pml_base_module_irecv_fn_t irecv; + mca_pml_base_module_recv_fn_t recv; + mca_pml_base_module_isend_init_fn_t isend_init; + mca_pml_base_module_isend_fn_t isend; + mca_pml_base_module_send_fn_t send; + mca_pml_base_module_iprobe_fn_t iprobe; + mca_pml_base_module_probe_fn_t probe; + mca_pml_base_module_start_fn_t start; + mca_pml_base_module_dump_fn_t dump; + + /* Custom requests classes to add extra data at end of pml requests */ + opal_class_t * req_recv_class; + opal_class_t * req_send_class; +} mca_vprotocol_base_module_1_0_0_t; +typedef mca_vprotocol_base_module_1_0_0_t mca_vprotocol_base_module_t; + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +/* silently include the pml_v.h as every file including vprotocol.h will also + * need it + */ +#include "pml_v.h" +#include "base/base.h" + +#endif /* __INCLUDE_VPROTOCOL_H_ */