2005-08-12 06:41:14 +04:00
|
|
|
/*
|
2007-03-17 02:11:45 +03:00
|
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
2005-11-05 22:57:48 +03:00
|
|
|
* University Research and Technology
|
|
|
|
* Corporation. All rights reserved.
|
|
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
* reserved.
|
2005-08-12 06:41:14 +04:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
|
|
* University of Stuttgart. All rights reserved.
|
2006-03-28 02:48:12 +04:00
|
|
|
* Copyright (c) 2004-2006 The Regents of the University of California.
|
2005-08-12 06:41:14 +04:00
|
|
|
* All rights reserved.
|
|
|
|
* $COPYRIGHT$
|
|
|
|
*
|
|
|
|
* Additional copyrights may follow
|
|
|
|
*
|
|
|
|
* $HEADER$
|
|
|
|
*/
|
|
|
|
/**
|
|
|
|
* @file
|
|
|
|
*
|
|
|
|
* BML Management Layer (BML)
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2005-09-13 00:22:59 +04:00
|
|
|
#ifndef MCA_BML_H
|
|
|
|
#define MCA_BML_H
|
|
|
|
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "opal/mca/mca.h"
|
|
|
|
#include "ompi/mca/btl/btl.h"
|
2005-08-12 06:41:14 +04:00
|
|
|
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "ompi/mca/bml/base/bml_base_btl.h"
|
|
|
|
#include "ompi/mca/bml/base/bml_base_endpoint.h"
|
2005-08-12 06:41:14 +04:00
|
|
|
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "ompi/types.h"
|
|
|
|
#include "ompi/class/ompi_free_list.h"
|
2005-08-12 06:41:14 +04:00
|
|
|
|
2007-03-17 02:11:45 +03:00
|
|
|
#include "opal/mca/crs/crs.h"
|
|
|
|
#include "opal/mca/crs/base/base.h"
|
|
|
|
|
2006-03-23 20:50:20 +03:00
|
|
|
#define OMPI_ENABLE_DEBUG_RELIABILITY 0
|
2006-03-17 21:46:48 +03:00
|
|
|
|
2005-08-12 06:41:14 +04:00
|
|
|
/*
|
|
|
|
* BML types
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct ompi_proc_t;
|
|
|
|
struct mca_bml_base_module_t;
|
|
|
|
struct mca_bml_base_endpoint_t;
|
|
|
|
struct mca_mpool_base_resources_t;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Cached set of information for each btl
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct mca_bml_base_btl_t {
|
|
|
|
int btl_index; /**< index in endpoint array */
|
|
|
|
int btl_flags; /**< support for put/get? */
|
2005-10-12 00:41:51 +04:00
|
|
|
double btl_weight; /**< BTL weight for scheduling */
|
2005-08-12 06:41:14 +04:00
|
|
|
size_t btl_eager_limit; /**< BTL eager limit */
|
2007-12-16 11:35:17 +03:00
|
|
|
size_t btl_rndv_eager_limit; /**< BTL rndv eager limit */
|
|
|
|
size_t btl_max_send_size; /**< BTL min send size */
|
2007-06-21 11:12:40 +04:00
|
|
|
size_t btl_rdma_pipeline_send_length; /**< BTL send length */
|
2007-05-17 11:54:27 +04:00
|
|
|
size_t btl_rdma_pipeline_frag_size; /**< BTL rdma frag size */
|
|
|
|
size_t btl_min_rdma_pipeline_size; /**< BTL min rdma size */
|
2005-08-12 06:41:14 +04:00
|
|
|
struct mca_btl_base_module_t *btl; /**< BTL module */
|
|
|
|
struct mca_btl_base_endpoint_t* btl_endpoint; /**< BTL addressing info */
|
|
|
|
|
|
|
|
/* BTL function table */
|
|
|
|
mca_btl_base_module_alloc_fn_t btl_alloc;
|
|
|
|
mca_btl_base_module_free_fn_t btl_free;
|
|
|
|
mca_btl_base_module_send_fn_t btl_send;
|
|
|
|
mca_btl_base_module_prepare_fn_t btl_prepare_src;
|
|
|
|
mca_btl_base_module_prepare_fn_t btl_prepare_dst;
|
|
|
|
mca_btl_base_module_put_fn_t btl_put;
|
|
|
|
mca_btl_base_module_get_fn_t btl_get;
|
|
|
|
mca_btl_base_component_progress_fn_t btl_progress;
|
2008-03-18 22:01:27 +03:00
|
|
|
|
|
|
|
mca_mpool_base_module_t* btl_mpool;
|
2005-08-12 06:41:14 +04:00
|
|
|
};
|
|
|
|
typedef struct mca_bml_base_btl_t mca_bml_base_btl_t;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A dynamically growable array of mca_bml_base_btl_t instances.
|
|
|
|
* Maintains an index into the array that is used for round-robin
|
|
|
|
* scheduling across contents.
|
|
|
|
*/
|
|
|
|
struct mca_bml_base_btl_array_t {
|
|
|
|
opal_object_t super;
|
|
|
|
size_t arr_size; /**< number available */
|
|
|
|
size_t arr_reserve; /**< size of allocated btl_proc array */
|
|
|
|
size_t arr_index; /**< last used index*/
|
|
|
|
mca_bml_base_btl_t* bml_btls; /**< array of bml btl's */
|
|
|
|
};
|
|
|
|
typedef struct mca_bml_base_btl_array_t mca_bml_base_btl_array_t;
|
|
|
|
|
|
|
|
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bml_base_btl_array_t);
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* If required, reallocate (grow) the array to the indicate size.
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
|
|
|
* @param size (IN)
|
|
|
|
*/
|
|
|
|
int mca_bml_base_btl_array_reserve(mca_bml_base_btl_array_t*, size_t);
|
|
|
|
|
|
|
|
static inline size_t mca_bml_base_btl_array_get_size(mca_bml_base_btl_array_t* array)
|
|
|
|
{
|
|
|
|
return array->arr_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Grow the array if required, and set the size.
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
|
|
|
* @param size (IN)
|
|
|
|
*/
|
|
|
|
static inline void mca_bml_base_btl_array_set_size(mca_bml_base_btl_array_t* array, size_t size)
|
|
|
|
{
|
|
|
|
if(array->arr_size > array->arr_reserve)
|
|
|
|
mca_bml_base_btl_array_reserve(array, size);
|
|
|
|
array->arr_size = size;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Grow the array size by one and return the item at that index.
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
|
|
|
*/
|
|
|
|
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_insert(mca_bml_base_btl_array_t* array)
|
|
|
|
{
|
|
|
|
#if OMPI_ENABLE_DEBUG
|
|
|
|
if(array->arr_size >= array->arr_reserve) {
|
This commit represents a bunch of work on a Mercurial side branch. As
such, the commit message back to the master SVN repository is fairly
long.
= ORTE Job-Level Output Messages =
Add two new interfaces that should be used for all new code throughout
the ORTE and OMPI layers (we already make the search-and-replace on
the existing ORTE / OMPI layers):
* orte_output(): (and corresponding friends ORTE_OUTPUT,
orte_output_verbose, etc.) This function sends the output directly
to the HNP for processing as part of a job-specific output
channel. It supports all the same outputs as opal_output()
(syslog, file, stdout, stderr), but for stdout/stderr, the output
is sent to the HNP for processing and output. More on this below.
* orte_show_help(): This function is a drop-in-replacement for
opal_show_help(), with two differences in functionality:
1. the rendered text help message output is sent to the HNP for
display (rather than outputting directly into the process' stderr
stream)
1. the HNP detects duplicate help messages and does not display them
(so that you don't see the same error message N times, once from
each of your N MPI processes); instead, it counts "new" instances
of the help message and displays a message every ~5 seconds when
there are new ones ("I got X new copies of the help message...")
opal_show_help and opal_output still exist, but they only output in
the current process. The intent for the new orte_* functions is that
they can apply job-level intelligence to the output. As such, we
recommend that all new ORTE and OMPI code use the new orte_*
functions, not thei opal_* functions.
=== New code ===
For ORTE and OMPI programmers, here's what you need to do differently
in new code:
* Do not include opal/util/show_help.h or opal/util/output.h.
Instead, include orte/util/output.h (this one header file has
declarations for both the orte_output() series of functions and
orte_show_help()).
* Effectively s/opal_output/orte_output/gi throughout your code.
Note that orte_output_open() takes a slightly different argument
list (as a way to pass data to the filtering stream -- see below),
so you if explicitly call opal_output_open(), you'll need to
slightly adapt to the new signature of orte_output_open().
* Literally s/opal_show_help/orte_show_help/. The function signature
is identical.
=== Notes ===
* orte_output'ing to stream 0 will do similar to what
opal_output'ing did, so leaving a hard-coded "0" as the first
argument is safe.
* For systems that do not use ORTE's RML or the HNP, the effect of
orte_output_* and orte_show_help will be identical to their opal
counterparts (the additional information passed to
orte_output_open() will be lost!). Indeed, the orte_* functions
simply become trivial wrappers to their opal_* counterparts. Note
that we have not tested this; the code is simple but it is quite
possible that we mucked something up.
= Filter Framework =
Messages sent view the new orte_* functions described above and
messages output via the IOF on the HNP will now optionally be passed
through a new "filter" framework before being output to
stdout/stderr. The "filter" OPAL MCA framework is intended to allow
preprocessing to messages before they are sent to their final
destinations. The first component that was written in the filter
framework was to create an XML stream, segregating all the messages
into different XML tags, etc. This will allow 3rd party tools to read
the stdout/stderr from the HNP and be able to know exactly what each
text message is (e.g., a help message, another OMPI infrastructure
message, stdout from the user process, stderr from the user process,
etc.).
Filtering is not active by default. Filter components must be
specifically requested, such as:
{{{
$ mpirun --mca filter xml ...
}}}
There can only be one filter component active.
= New MCA Parameters =
The new functionality described above introduces two new MCA
parameters:
* '''orte_base_help_aggregate''': Defaults to 1 (true), meaning that
help messages will be aggregated, as described above. If set to 0,
all help messages will be displayed, even if they are duplicates
(i.e., the original behavior).
* '''orte_base_show_output_recursions''': An MCA parameter to help
debug one of the known issues, described below. It is likely that
this MCA parameter will disappear before v1.3 final.
= Known Issues =
* The XML filter component is not complete. The current output from
this component is preliminary and not real XML. A bit more work
needs to be done to configure.m4 search for an appropriate XML
library/link it in/use it at run time.
* There are possible recursion loops in the orte_output() and
orte_show_help() functions -- e.g., if RML send calls orte_output()
or orte_show_help(). We have some ideas how to fix these, but
figured that it was ok to commit before feature freeze with known
issues. The code currently contains sub-optimal workarounds so
that this will not be a problem, but it would be good to actually
solve the problem rather than have hackish workarounds before v1.3 final.
This commit was SVN r18434.
2008-05-14 00:00:55 +04:00
|
|
|
orte_output(0, "mca_bml_base_btl_array_insert: invalid array index %lu >= %lu",
|
2007-01-31 07:27:41 +03:00
|
|
|
(unsigned long)array->arr_size, (unsigned long)array->arr_reserve);
|
2005-08-12 06:41:14 +04:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return &array->bml_btls[array->arr_size++];
|
|
|
|
}
|
|
|
|
|
2006-05-20 06:39:05 +04:00
|
|
|
/**
|
|
|
|
* Remove a btl from a bml_btl
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
|
|
|
* @param btl (IN)
|
|
|
|
*/
|
2006-05-21 09:53:22 +04:00
|
|
|
static inline bool mca_bml_base_btl_array_remove( mca_bml_base_btl_array_t* array,
|
|
|
|
struct mca_btl_base_module_t* btl )
|
|
|
|
{
|
|
|
|
size_t i = 0;
|
2006-05-20 06:39:05 +04:00
|
|
|
/* find the btl */
|
2006-05-21 09:53:22 +04:00
|
|
|
for( i = 0; i < array->arr_size; i++ ) {
|
|
|
|
if( array->bml_btls[i].btl == btl ) {
|
2007-04-23 01:43:34 +04:00
|
|
|
/* make sure not to go out of bounds */
|
|
|
|
for( ; i < array->arr_size-1; i++ ) {
|
2006-05-21 09:53:22 +04:00
|
|
|
/* move all btl's back by 1, so the found
|
|
|
|
btl is "removed" */
|
|
|
|
array->bml_btls[i] = array->bml_btls[(i+1)];
|
|
|
|
}
|
|
|
|
array->arr_size--;
|
|
|
|
array->arr_index = 0;
|
|
|
|
return true;
|
2006-05-20 06:39:05 +04:00
|
|
|
}
|
|
|
|
}
|
2006-05-21 09:53:22 +04:00
|
|
|
return false;
|
2006-05-20 06:39:05 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-08-12 06:41:14 +04:00
|
|
|
/**
|
|
|
|
* Return an array item at the specified index.
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
2007-01-19 22:48:06 +03:00
|
|
|
* @param item_index (IN)
|
2005-08-12 06:41:14 +04:00
|
|
|
*/
|
2007-01-19 22:48:06 +03:00
|
|
|
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_get_index(mca_bml_base_btl_array_t* array, size_t item_index)
|
2005-08-12 06:41:14 +04:00
|
|
|
{
|
|
|
|
#if OMPI_ENABLE_DEBUG
|
2007-01-19 22:48:06 +03:00
|
|
|
if(item_index >= array->arr_size) {
|
This commit represents a bunch of work on a Mercurial side branch. As
such, the commit message back to the master SVN repository is fairly
long.
= ORTE Job-Level Output Messages =
Add two new interfaces that should be used for all new code throughout
the ORTE and OMPI layers (we already make the search-and-replace on
the existing ORTE / OMPI layers):
* orte_output(): (and corresponding friends ORTE_OUTPUT,
orte_output_verbose, etc.) This function sends the output directly
to the HNP for processing as part of a job-specific output
channel. It supports all the same outputs as opal_output()
(syslog, file, stdout, stderr), but for stdout/stderr, the output
is sent to the HNP for processing and output. More on this below.
* orte_show_help(): This function is a drop-in-replacement for
opal_show_help(), with two differences in functionality:
1. the rendered text help message output is sent to the HNP for
display (rather than outputting directly into the process' stderr
stream)
1. the HNP detects duplicate help messages and does not display them
(so that you don't see the same error message N times, once from
each of your N MPI processes); instead, it counts "new" instances
of the help message and displays a message every ~5 seconds when
there are new ones ("I got X new copies of the help message...")
opal_show_help and opal_output still exist, but they only output in
the current process. The intent for the new orte_* functions is that
they can apply job-level intelligence to the output. As such, we
recommend that all new ORTE and OMPI code use the new orte_*
functions, not thei opal_* functions.
=== New code ===
For ORTE and OMPI programmers, here's what you need to do differently
in new code:
* Do not include opal/util/show_help.h or opal/util/output.h.
Instead, include orte/util/output.h (this one header file has
declarations for both the orte_output() series of functions and
orte_show_help()).
* Effectively s/opal_output/orte_output/gi throughout your code.
Note that orte_output_open() takes a slightly different argument
list (as a way to pass data to the filtering stream -- see below),
so you if explicitly call opal_output_open(), you'll need to
slightly adapt to the new signature of orte_output_open().
* Literally s/opal_show_help/orte_show_help/. The function signature
is identical.
=== Notes ===
* orte_output'ing to stream 0 will do similar to what
opal_output'ing did, so leaving a hard-coded "0" as the first
argument is safe.
* For systems that do not use ORTE's RML or the HNP, the effect of
orte_output_* and orte_show_help will be identical to their opal
counterparts (the additional information passed to
orte_output_open() will be lost!). Indeed, the orte_* functions
simply become trivial wrappers to their opal_* counterparts. Note
that we have not tested this; the code is simple but it is quite
possible that we mucked something up.
= Filter Framework =
Messages sent view the new orte_* functions described above and
messages output via the IOF on the HNP will now optionally be passed
through a new "filter" framework before being output to
stdout/stderr. The "filter" OPAL MCA framework is intended to allow
preprocessing to messages before they are sent to their final
destinations. The first component that was written in the filter
framework was to create an XML stream, segregating all the messages
into different XML tags, etc. This will allow 3rd party tools to read
the stdout/stderr from the HNP and be able to know exactly what each
text message is (e.g., a help message, another OMPI infrastructure
message, stdout from the user process, stderr from the user process,
etc.).
Filtering is not active by default. Filter components must be
specifically requested, such as:
{{{
$ mpirun --mca filter xml ...
}}}
There can only be one filter component active.
= New MCA Parameters =
The new functionality described above introduces two new MCA
parameters:
* '''orte_base_help_aggregate''': Defaults to 1 (true), meaning that
help messages will be aggregated, as described above. If set to 0,
all help messages will be displayed, even if they are duplicates
(i.e., the original behavior).
* '''orte_base_show_output_recursions''': An MCA parameter to help
debug one of the known issues, described below. It is likely that
this MCA parameter will disappear before v1.3 final.
= Known Issues =
* The XML filter component is not complete. The current output from
this component is preliminary and not real XML. A bit more work
needs to be done to configure.m4 search for an appropriate XML
library/link it in/use it at run time.
* There are possible recursion loops in the orte_output() and
orte_show_help() functions -- e.g., if RML send calls orte_output()
or orte_show_help(). We have some ideas how to fix these, but
figured that it was ok to commit before feature freeze with known
issues. The code currently contains sub-optimal workarounds so
that this will not be a problem, but it would be good to actually
solve the problem rather than have hackish workarounds before v1.3 final.
This commit was SVN r18434.
2008-05-14 00:00:55 +04:00
|
|
|
orte_output(0, "mca_bml_base_btl_array_get_index: invalid array index %lu >= %lu",
|
2007-01-31 07:27:41 +03:00
|
|
|
(unsigned long)item_index, (unsigned long)array->arr_size);
|
2005-08-12 06:41:14 +04:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
2007-01-19 22:48:06 +03:00
|
|
|
return &array->bml_btls[item_index];
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Return the next LRU index in the array.
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
2007-01-19 22:48:06 +03:00
|
|
|
*
|
|
|
|
* @param index (OUT)
|
2005-08-12 06:41:14 +04:00
|
|
|
*/
|
|
|
|
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_get_next(mca_bml_base_btl_array_t* array)
|
|
|
|
{
|
|
|
|
#if OMPI_ENABLE_DEBUG
|
|
|
|
if(array->arr_size == 0) {
|
This commit represents a bunch of work on a Mercurial side branch. As
such, the commit message back to the master SVN repository is fairly
long.
= ORTE Job-Level Output Messages =
Add two new interfaces that should be used for all new code throughout
the ORTE and OMPI layers (we already make the search-and-replace on
the existing ORTE / OMPI layers):
* orte_output(): (and corresponding friends ORTE_OUTPUT,
orte_output_verbose, etc.) This function sends the output directly
to the HNP for processing as part of a job-specific output
channel. It supports all the same outputs as opal_output()
(syslog, file, stdout, stderr), but for stdout/stderr, the output
is sent to the HNP for processing and output. More on this below.
* orte_show_help(): This function is a drop-in-replacement for
opal_show_help(), with two differences in functionality:
1. the rendered text help message output is sent to the HNP for
display (rather than outputting directly into the process' stderr
stream)
1. the HNP detects duplicate help messages and does not display them
(so that you don't see the same error message N times, once from
each of your N MPI processes); instead, it counts "new" instances
of the help message and displays a message every ~5 seconds when
there are new ones ("I got X new copies of the help message...")
opal_show_help and opal_output still exist, but they only output in
the current process. The intent for the new orte_* functions is that
they can apply job-level intelligence to the output. As such, we
recommend that all new ORTE and OMPI code use the new orte_*
functions, not thei opal_* functions.
=== New code ===
For ORTE and OMPI programmers, here's what you need to do differently
in new code:
* Do not include opal/util/show_help.h or opal/util/output.h.
Instead, include orte/util/output.h (this one header file has
declarations for both the orte_output() series of functions and
orte_show_help()).
* Effectively s/opal_output/orte_output/gi throughout your code.
Note that orte_output_open() takes a slightly different argument
list (as a way to pass data to the filtering stream -- see below),
so you if explicitly call opal_output_open(), you'll need to
slightly adapt to the new signature of orte_output_open().
* Literally s/opal_show_help/orte_show_help/. The function signature
is identical.
=== Notes ===
* orte_output'ing to stream 0 will do similar to what
opal_output'ing did, so leaving a hard-coded "0" as the first
argument is safe.
* For systems that do not use ORTE's RML or the HNP, the effect of
orte_output_* and orte_show_help will be identical to their opal
counterparts (the additional information passed to
orte_output_open() will be lost!). Indeed, the orte_* functions
simply become trivial wrappers to their opal_* counterparts. Note
that we have not tested this; the code is simple but it is quite
possible that we mucked something up.
= Filter Framework =
Messages sent view the new orte_* functions described above and
messages output via the IOF on the HNP will now optionally be passed
through a new "filter" framework before being output to
stdout/stderr. The "filter" OPAL MCA framework is intended to allow
preprocessing to messages before they are sent to their final
destinations. The first component that was written in the filter
framework was to create an XML stream, segregating all the messages
into different XML tags, etc. This will allow 3rd party tools to read
the stdout/stderr from the HNP and be able to know exactly what each
text message is (e.g., a help message, another OMPI infrastructure
message, stdout from the user process, stderr from the user process,
etc.).
Filtering is not active by default. Filter components must be
specifically requested, such as:
{{{
$ mpirun --mca filter xml ...
}}}
There can only be one filter component active.
= New MCA Parameters =
The new functionality described above introduces two new MCA
parameters:
* '''orte_base_help_aggregate''': Defaults to 1 (true), meaning that
help messages will be aggregated, as described above. If set to 0,
all help messages will be displayed, even if they are duplicates
(i.e., the original behavior).
* '''orte_base_show_output_recursions''': An MCA parameter to help
debug one of the known issues, described below. It is likely that
this MCA parameter will disappear before v1.3 final.
= Known Issues =
* The XML filter component is not complete. The current output from
this component is preliminary and not real XML. A bit more work
needs to be done to configure.m4 search for an appropriate XML
library/link it in/use it at run time.
* There are possible recursion loops in the orte_output() and
orte_show_help() functions -- e.g., if RML send calls orte_output()
or orte_show_help(). We have some ideas how to fix these, but
figured that it was ok to commit before feature freeze with known
issues. The code currently contains sub-optimal workarounds so
that this will not be a problem, but it would be good to actually
solve the problem rather than have hackish workarounds before v1.3 final.
This commit was SVN r18434.
2008-05-14 00:00:55 +04:00
|
|
|
orte_output(0, "mca_bml_base_btl_array_get_next: invalid array size");
|
2005-08-12 06:41:14 +04:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
2006-05-23 03:28:40 +04:00
|
|
|
if( 1 == array->arr_size ) {
|
|
|
|
return &array->bml_btls[0]; /* force the return to avoid a jump */
|
|
|
|
} else {
|
2006-10-20 07:57:44 +04:00
|
|
|
size_t current_position = array->arr_index; /* force to always start from zero */
|
2006-05-23 03:28:40 +04:00
|
|
|
if( (current_position + 1) == array->arr_size ) {
|
|
|
|
array->arr_index = 0; /* next time serve from the beginning */
|
|
|
|
} else {
|
|
|
|
array->arr_index = current_position + 1; /* continue */
|
|
|
|
}
|
|
|
|
return &array->bml_btls[current_position];
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Locate an element in the array
|
|
|
|
*
|
|
|
|
* @param array (IN)
|
|
|
|
* @param index (IN)
|
|
|
|
*/
|
|
|
|
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_find(
|
|
|
|
mca_bml_base_btl_array_t* array, struct mca_btl_base_module_t* btl)
|
|
|
|
{
|
|
|
|
size_t i=0;
|
|
|
|
for(i=0; i<array->arr_size; i++) {
|
|
|
|
if(array->bml_btls[i].btl == btl) {
|
|
|
|
return &array->bml_btls[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Structure associated w/ ompi_proc_t that contains the set
|
2006-03-29 20:19:17 +04:00
|
|
|
* of BTLs used to reach a destination
|
2005-08-12 06:41:14 +04:00
|
|
|
*/
|
|
|
|
struct mca_bml_base_endpoint_t {
|
2006-07-04 05:20:20 +04:00
|
|
|
opal_list_item_t super; /**< base_endpoint is a list item */
|
|
|
|
struct ompi_proc_t* btl_proc; /**< backpointer to target ompi_proc_t */
|
2007-06-21 11:12:40 +04:00
|
|
|
size_t btl_pipeline_send_length; /**< max of pipeline send_length of available BTLs */
|
2007-05-17 11:54:27 +04:00
|
|
|
size_t btl_send_limit; /**< max of min rdma pipeline for available rmda btls */
|
2005-12-21 18:11:40 +03:00
|
|
|
size_t btl_max_send_size; /**< min of max send size for available send btls */
|
2006-02-09 18:49:51 +03:00
|
|
|
size_t btl_rdma_align; /**< max of min rdma size for available rmda btls */
|
2005-12-21 18:11:40 +03:00
|
|
|
mca_bml_base_btl_array_t btl_eager; /**< array of btls to use for first fragments */
|
|
|
|
mca_bml_base_btl_array_t btl_send; /**< array of btls to use for remaining fragments */
|
2006-08-20 19:54:04 +04:00
|
|
|
size_t bml_max_send_length;
|
|
|
|
size_t bml_max_rdma_length;
|
2005-12-21 18:11:40 +03:00
|
|
|
mca_bml_base_btl_array_t btl_rdma; /**< array of btls that support (prefer) rdma */
|
2007-01-18 12:15:18 +03:00
|
|
|
size_t btl_rdma_index; /**< index of last used BTL for RDMA */
|
2006-06-28 00:23:47 +04:00
|
|
|
uint32_t btl_flags_or; /**< the bitwise OR of the btl flags */
|
|
|
|
uint32_t btl_flags_and; /**< the bitwise AND of the btl flags */
|
2005-08-12 06:41:14 +04:00
|
|
|
};
|
|
|
|
typedef struct mca_bml_base_endpoint_t mca_bml_base_endpoint_t;
|
|
|
|
|
|
|
|
|
2006-08-20 19:54:04 +04:00
|
|
|
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bml_base_endpoint_t);
|
2005-08-12 06:41:14 +04:00
|
|
|
|
2007-12-09 17:08:01 +03:00
|
|
|
static inline void mca_bml_base_alloc(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t** des, uint8_t order, size_t size, uint32_t flags) {
|
|
|
|
*des = bml_btl->btl_alloc(bml_btl->btl, bml_btl->btl_endpoint, order, size, flags);
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void mca_bml_base_free(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t* des) {
|
|
|
|
bml_btl->btl_free( bml_btl->btl, des );
|
2005-11-14 02:07:37 +03:00
|
|
|
/* The previous function is supposed to release the des object
|
|
|
|
* so we should not touch it anymore.
|
|
|
|
*/
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
|
2006-03-17 21:46:48 +03:00
|
|
|
#if OMPI_ENABLE_DEBUG_RELIABILITY
|
|
|
|
|
2006-03-21 00:58:20 +03:00
|
|
|
int mca_bml_base_send(
|
2006-03-17 21:46:48 +03:00
|
|
|
mca_bml_base_btl_t* bml_btl,
|
|
|
|
mca_btl_base_descriptor_t* des,
|
2006-03-21 00:58:20 +03:00
|
|
|
mca_btl_base_tag_t tag);
|
|
|
|
|
2006-03-17 21:46:48 +03:00
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
static inline int mca_bml_base_send(
|
2007-12-09 17:12:38 +03:00
|
|
|
mca_bml_base_btl_t* bml_btl,
|
|
|
|
mca_btl_base_descriptor_t* des,
|
|
|
|
mca_btl_base_tag_t tag)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
des->des_context = (void*) bml_btl;
|
|
|
|
rc = bml_btl->btl_send(bml_btl->btl, bml_btl->btl_endpoint, des, tag);
|
|
|
|
if(rc == OMPI_ERR_RESOURCE_BUSY)
|
|
|
|
rc = OMPI_SUCCESS;
|
|
|
|
|
|
|
|
return rc;
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
|
2006-03-17 21:46:48 +03:00
|
|
|
#endif
|
|
|
|
|
2007-12-09 17:12:38 +03:00
|
|
|
static inline int mca_bml_base_send_status(
|
|
|
|
mca_bml_base_btl_t* bml_btl,
|
|
|
|
mca_btl_base_descriptor_t* des,
|
|
|
|
mca_btl_base_tag_t tag)
|
|
|
|
{
|
|
|
|
des->des_context = (void*) bml_btl;
|
|
|
|
return bml_btl->btl_send(bml_btl->btl, bml_btl->btl_endpoint, des, tag);
|
|
|
|
}
|
|
|
|
|
2005-08-12 06:41:14 +04:00
|
|
|
static inline int mca_bml_base_put(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t* des) {
|
|
|
|
des->des_context = (void*) bml_btl;
|
|
|
|
return bml_btl->btl_put(
|
|
|
|
bml_btl->btl,
|
|
|
|
bml_btl->btl_endpoint,
|
|
|
|
des);
|
|
|
|
}
|
|
|
|
|
2005-08-17 22:23:38 +04:00
|
|
|
static inline int mca_bml_base_get(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t* des) {
|
|
|
|
des->des_context = (void*) bml_btl;
|
|
|
|
return bml_btl->btl_get(
|
|
|
|
bml_btl->btl,
|
|
|
|
bml_btl->btl_endpoint,
|
|
|
|
des);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-08-12 06:41:14 +04:00
|
|
|
static inline void mca_bml_base_prepare_src(mca_bml_base_btl_t* bml_btl,
|
|
|
|
mca_mpool_base_registration_t* reg,
|
|
|
|
struct ompi_convertor_t* conv,
|
2007-05-24 23:51:26 +04:00
|
|
|
uint8_t order,
|
2005-08-12 06:41:14 +04:00
|
|
|
size_t reserve,
|
2007-05-24 23:51:26 +04:00
|
|
|
size_t *size,
|
2007-12-09 17:08:01 +03:00
|
|
|
uint32_t flags,
|
2005-08-12 06:41:14 +04:00
|
|
|
mca_btl_base_descriptor_t** des) {
|
2008-03-18 22:01:27 +03:00
|
|
|
*des = bml_btl->btl_prepare_src( bml_btl->btl,
|
|
|
|
bml_btl->btl_endpoint,
|
|
|
|
reg,
|
|
|
|
conv,
|
|
|
|
order,
|
|
|
|
reserve,
|
|
|
|
size,
|
|
|
|
flags );
|
|
|
|
if( OPAL_LIKELY((*des) != NULL) ) {
|
|
|
|
(*des)->des_context = (void*) bml_btl;
|
|
|
|
}
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void mca_bml_base_prepare_dst(mca_bml_base_btl_t* bml_btl,
|
|
|
|
mca_mpool_base_registration_t* reg,
|
|
|
|
struct ompi_convertor_t* conv,
|
2007-05-24 23:51:26 +04:00
|
|
|
uint8_t order,
|
2005-08-12 06:41:14 +04:00
|
|
|
size_t reserve,
|
2007-12-09 17:08:01 +03:00
|
|
|
size_t *size,
|
|
|
|
uint32_t flags,
|
2005-08-12 06:41:14 +04:00
|
|
|
mca_btl_base_descriptor_t** des) {
|
2008-03-18 22:01:27 +03:00
|
|
|
*des = bml_btl->btl_prepare_dst( bml_btl->btl,
|
|
|
|
bml_btl->btl_endpoint,
|
|
|
|
reg,
|
|
|
|
conv,
|
|
|
|
order,
|
|
|
|
reserve,
|
|
|
|
size,
|
|
|
|
flags );
|
|
|
|
if( OPAL_LIKELY((*des) != NULL) ) {
|
|
|
|
(*des)->des_context = (void*) bml_btl;
|
|
|
|
}
|
2005-08-12 06:41:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* BML component interface functions and datatype.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* MCA->BML Initializes the BML component and creates specific BML
|
|
|
|
* module(s).
|
|
|
|
*
|
|
|
|
* @param num_bmls (OUT) Returns the number of bml modules created, or 0
|
|
|
|
* if the transport is not available.
|
|
|
|
*
|
|
|
|
* @param enable_progress_threads (IN) Whether this component is
|
|
|
|
* allowed to run a hidden/progress thread or not.
|
|
|
|
*
|
|
|
|
* @param enable_mpi_threads (IN) Whether support for multiple MPI
|
|
|
|
* threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which
|
|
|
|
* indicates whether multiple threads may invoke this component
|
|
|
|
* simultaneously or not.
|
|
|
|
*
|
|
|
|
* @return Array of pointers to BML modules, or NULL if the transport
|
|
|
|
* is not available.
|
|
|
|
*
|
|
|
|
* During component initialization, the BML component should discover
|
|
|
|
* the physical devices that are available for the given transport,
|
|
|
|
* and create a BML module to represent each device. Any addressing
|
|
|
|
* information required by peers to reach the device should be published
|
|
|
|
* during this function via the mca_base_modex_send() interface.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
typedef struct mca_bml_base_module_t* (*mca_bml_base_component_init_fn_t)(
|
|
|
|
int* priority,
|
|
|
|
bool enable_progress_threads,
|
2006-07-04 05:20:20 +04:00
|
|
|
bool enable_mpi_threads
|
2005-08-12 06:41:14 +04:00
|
|
|
);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* MCA->BML Called to progress outstanding requests for
|
|
|
|
* non-threaded polling environments.
|
|
|
|
*
|
|
|
|
* @param tstamp Current time.
|
|
|
|
* @return OMPI_SUCCESS or error code on failure.
|
|
|
|
*/
|
|
|
|
|
|
|
|
typedef int (*mca_bml_base_module_progress_fn_t)(void);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* BML component descriptor. Contains component version information
|
|
|
|
* and component open/close/init functions.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct mca_bml_base_component_1_0_0_t {
|
|
|
|
mca_base_component_t bml_version;
|
|
|
|
mca_base_component_data_1_0_0_t bml_data;
|
|
|
|
mca_bml_base_component_init_fn_t bml_init;
|
|
|
|
};
|
|
|
|
typedef struct mca_bml_base_component_1_0_0_t mca_bml_base_component_1_0_0_t;
|
|
|
|
typedef struct mca_bml_base_component_1_0_0_t mca_bml_base_component_t;
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* BML module interface functions and datatype.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* MCA->BML Clean up any resources held by BML module
|
|
|
|
* before the module is unloaded.
|
|
|
|
*
|
|
|
|
* @param bml (IN) BML module.
|
|
|
|
*
|
|
|
|
* Prior to unloading a BML module, the MCA framework will call
|
|
|
|
* the BML finalize method of the module. Any resources held by
|
|
|
|
* the BML should be released and if required the memory corresponding
|
|
|
|
* to the BML module freed.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_finalize_fn_t)( void );
|
|
|
|
|
|
|
|
/**
|
|
|
|
* PML->BML notification of change in the process list.
|
|
|
|
*
|
|
|
|
* @param nprocs (IN) Number of processes
|
|
|
|
* @param procs (IN) Set of processes
|
|
|
|
* @param endpoint (OUT) Set of (optional) mca_bml_base_endpoint_t structures by BML.
|
|
|
|
* @param reachable (OUT) Bitmask indicating set of peer processes that are reachable by this BML.
|
|
|
|
* @return OMPI_SUCCESS or error status on failure.
|
|
|
|
*
|
|
|
|
* The mca_bml_base_module_add_procs_fn_t() is called by the PML to
|
|
|
|
* determine the set of BMLs that should be used to reach each process.
|
|
|
|
* Any addressing information exported by the peer via the mca_base_modex_send()
|
|
|
|
* function should be available during this call via the corresponding
|
|
|
|
* mca_base_modex_recv() function. The BML may utilize this information to
|
|
|
|
* determine reachability of each peer process.
|
|
|
|
*
|
|
|
|
* For each process that is reachable by the BML, the bit corresponding to the index
|
|
|
|
* into the proc array (nprocs) should be set in the reachable bitmask. The PML
|
|
|
|
* provides the BML the option to return a pointer to a data structure defined
|
|
|
|
* by the BML that is returned to the BML on subsequent calls to the BML data
|
|
|
|
* transfer functions (e.g bml_send). This may be used by the BML to cache any addressing
|
|
|
|
* or connection information (e.g. TCP socket, IP queue pair).
|
2006-08-14 23:17:36 +04:00
|
|
|
*
|
|
|
|
* \note This function will return OMPI_ERR_UNREACH if one or more
|
|
|
|
* processes can not be reached by the currently active BTLs. This is
|
|
|
|
* not a fatal error, and the calling layer is free to continue using
|
|
|
|
* the BML interface.
|
2005-08-12 06:41:14 +04:00
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_add_procs_fn_t)(
|
|
|
|
size_t nprocs,
|
|
|
|
struct ompi_proc_t** procs,
|
|
|
|
struct mca_bml_base_endpoint_t** endpoints,
|
|
|
|
struct ompi_bitmap_t* reachable
|
|
|
|
);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Notification of change to the process list.
|
|
|
|
*
|
|
|
|
* @param nprocs (IN) Number of processes
|
|
|
|
* @param proc (IN) Set of processes
|
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
|
|
|
* When the process list changes, the PML notifies the BML of the
|
|
|
|
* change, to provide the opportunity to cleanup or release any
|
|
|
|
* resources associated with the peer.
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_del_procs_fn_t)(
|
|
|
|
size_t nprocs,
|
|
|
|
struct ompi_proc_t** procs
|
|
|
|
);
|
|
|
|
|
2006-03-28 02:48:12 +04:00
|
|
|
/**
|
|
|
|
* Notification of change to the btl list.
|
|
|
|
*
|
|
|
|
* @param bml (IN) BTL module
|
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
|
|
|
* On recovery of a btl, add it to the set of forwarding
|
|
|
|
* entries used by the BML.
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_add_btl_fn_t)( struct mca_btl_base_module_t* );
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Notification of change to the btl list.
|
|
|
|
*
|
|
|
|
* @param bml (IN) BTL module
|
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
|
|
|
* On failure of a btl, remove it from the set of forwarding
|
|
|
|
* entries used by the BML.
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_del_btl_fn_t)( struct mca_btl_base_module_t* );
|
|
|
|
|
2006-05-04 19:19:12 +04:00
|
|
|
/**
|
|
|
|
* Notification of change to the btl list.
|
|
|
|
*
|
|
|
|
* @param bml (IN) BTL module
|
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
|
|
|
* On failure of a btl, remove it from the set of forwarding
|
|
|
|
* entries used by the BML.
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_del_proc_btl_fn_t)(
|
|
|
|
struct ompi_proc_t*,
|
|
|
|
struct mca_btl_base_module_t* );
|
|
|
|
|
2005-08-12 06:41:14 +04:00
|
|
|
/**
|
|
|
|
* Register a callback function that is called on receipt
|
|
|
|
* of a fragment.
|
|
|
|
*
|
|
|
|
* @param bml (IN) BML module
|
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
|
|
|
* When the process list changes, the PML notifies the BML of the
|
|
|
|
* change, to provide the opportunity to cleanup or release any
|
|
|
|
* resources associated with the peer.
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_register_fn_t)(
|
|
|
|
mca_btl_base_tag_t tag,
|
2008-01-15 08:32:53 +03:00
|
|
|
mca_btl_base_module_recv_cb_fn_t cbfunc,
|
2005-08-12 06:41:14 +04:00
|
|
|
void* cbdata
|
|
|
|
);
|
2006-08-17 00:21:38 +04:00
|
|
|
/**
|
|
|
|
* Register a callback function that is called of error.
|
|
|
|
*
|
|
|
|
* @param bml (IN) BML module
|
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_register_error_cb_fn_t)(
|
|
|
|
mca_btl_base_module_error_cb_fn_t cbfunc
|
|
|
|
);
|
|
|
|
|
2007-03-17 02:11:45 +03:00
|
|
|
/**
|
|
|
|
* Fault Tolerance Event Notification Function
|
|
|
|
* @param status Checkpoint Status
|
|
|
|
* @return OMPI_SUCCESS or failure status
|
|
|
|
*/
|
|
|
|
typedef int (*mca_bml_base_module_ft_event_fn_t)(int status);
|
2006-08-17 00:21:38 +04:00
|
|
|
|
|
|
|
|
2005-08-12 06:41:14 +04:00
|
|
|
/**
|
|
|
|
* BML module interface functions and attributes.
|
|
|
|
*/
|
|
|
|
struct mca_bml_base_module_t {
|
|
|
|
/* BML common attributes */
|
|
|
|
mca_bml_base_component_t* bml_component; /**< pointer back to the BML component structure */
|
|
|
|
size_t bml_eager_limit; /**< maximum size of first fragment -- eager send */
|
2007-12-16 11:35:17 +03:00
|
|
|
size_t bml_rndv_eager_limit; /**< size of a first fragment of rndv protocol */
|
2005-08-12 06:41:14 +04:00
|
|
|
size_t bml_max_send_size; /**< maximum send fragment size supported by the BML */
|
|
|
|
size_t bml_min_rdma_size; /**< threshold below which the BML should not fragment */
|
|
|
|
size_t bml_max_rdma_size; /**< maximum rdma fragment size supported by the BML */
|
|
|
|
|
|
|
|
/* BML function table */
|
2006-05-04 19:19:12 +04:00
|
|
|
mca_bml_base_module_add_procs_fn_t bml_add_procs;
|
|
|
|
mca_bml_base_module_del_procs_fn_t bml_del_procs;
|
|
|
|
mca_bml_base_module_add_btl_fn_t bml_add_btl;
|
|
|
|
mca_bml_base_module_del_btl_fn_t bml_del_btl;
|
|
|
|
mca_bml_base_module_del_proc_btl_fn_t bml_del_proc_btl;
|
|
|
|
mca_bml_base_module_register_fn_t bml_register;
|
2006-08-17 00:21:38 +04:00
|
|
|
mca_bml_base_module_register_error_cb_fn_t bml_register_error;
|
|
|
|
|
2006-05-04 19:19:12 +04:00
|
|
|
mca_bml_base_module_finalize_fn_t bml_finalize;
|
2005-08-12 06:41:14 +04:00
|
|
|
|
|
|
|
mca_bml_base_module_progress_fn_t bml_progress;
|
|
|
|
|
2007-03-17 02:11:45 +03:00
|
|
|
mca_bml_base_module_ft_event_fn_t bml_ft_event;
|
2005-08-12 06:41:14 +04:00
|
|
|
};
|
|
|
|
typedef struct mca_bml_base_module_t mca_bml_base_module_t;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Macro for use in modules that are of type bml v1.0.0
|
|
|
|
*/
|
|
|
|
#define MCA_BML_BASE_VERSION_1_0_0 \
|
|
|
|
/* v1.0 is chained to MCA v1.0 */ \
|
|
|
|
MCA_BASE_VERSION_1_0_0, \
|
|
|
|
/* bml v1.0 */ \
|
|
|
|
"bml", 1, 0, 0
|
|
|
|
|
|
|
|
#endif /* OMPI_MCA_BML_H */
|