307 строки
9.6 KiB
C
307 строки
9.6 KiB
C
/**
|
|
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
|
|
$COPYRIGHT$
|
|
|
|
Additional copyrights may follow
|
|
|
|
$HEADER$
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <stdio.h>
|
|
|
|
#include <dlfcn.h>
|
|
#include <libgen.h>
|
|
|
|
#include "coll_fca.h"
|
|
|
|
/*
|
|
* Public string showing the coll ompi_fca component version number
|
|
*/
|
|
const char *mca_coll_fca_component_version_string =
|
|
"Open MPI FCA collective MCA component version " OMPI_VERSION;
|
|
|
|
/*
|
|
* Global variable
|
|
*/
|
|
int mca_coll_fca_output = -1;
|
|
|
|
/*
|
|
* Instantiate the public struct with all of our public information
|
|
* and pointers to our public functions in it
|
|
*/
|
|
static int fca_open(void);
|
|
static int fca_close(void);
|
|
static int fca_register(void);
|
|
|
|
mca_coll_fca_component_t mca_coll_fca_component = {
|
|
|
|
/* First, the mca_component_t struct containing meta information
|
|
about the component itfca */
|
|
{
|
|
{
|
|
MCA_COLL_BASE_VERSION_2_0_0,
|
|
|
|
/* Component name and version */
|
|
"fca",
|
|
OMPI_MAJOR_VERSION,
|
|
OMPI_MINOR_VERSION,
|
|
OMPI_RELEASE_VERSION,
|
|
|
|
/* Component open and close functions */
|
|
fca_open,
|
|
fca_close,
|
|
NULL,
|
|
fca_register
|
|
},
|
|
{
|
|
/* The component is not checkpoint ready */
|
|
MCA_BASE_METADATA_PARAM_NONE
|
|
},
|
|
|
|
/* Initialization / querying functions */
|
|
|
|
mca_coll_fca_init_query,
|
|
mca_coll_fca_comm_query,
|
|
}
|
|
};
|
|
|
|
#define FCA_API_CLEAR_MICRO(__x) ((__x>>FCA_MINOR_BIT)<<FCA_MINOR_BIT)
|
|
#define FCA_API_VER(__major,__minor) (__major<<FCA_MAJOR_BIT | __minor<<FCA_MINOR_BIT)
|
|
|
|
/**
|
|
* Called from FCA blocking functions to progress MPI
|
|
*/
|
|
static void mca_coll_fca_progress_cb(void *arg)
|
|
{
|
|
opal_progress();
|
|
}
|
|
|
|
/**
|
|
* Called from MPI blocking functions to progress FCA
|
|
*/
|
|
static int mca_coll_fca_mpi_progress_cb(void)
|
|
{
|
|
#ifdef OMPI_FCA_PROGRESS
|
|
if (!mca_coll_fca_component.fca_context)
|
|
return 0;
|
|
|
|
fca_progress(mca_coll_fca_component.fca_context);
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Initialize translation tables for FCA datatypes and operations
|
|
*/
|
|
static void mca_coll_fca_init_fca_translations(void)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < FCA_DT_MAX_PREDEFINED; ++i) {
|
|
mca_coll_fca_component.fca_dtypes[i].mpi_dtype = MPI_DATATYPE_NULL;
|
|
mca_coll_fca_component.fca_dtypes[i].fca_dtype = -1;
|
|
mca_coll_fca_component.fca_dtypes[i].fca_dtype_extent = 0;
|
|
}
|
|
|
|
for (i = 0; i < FCA_MAX_OPS; ++i) {
|
|
mca_coll_fca_component.fca_reduce_ops[i].mpi_op = MPI_OP_NULL;
|
|
mca_coll_fca_component.fca_reduce_ops[i].fca_op = -1;
|
|
}
|
|
}
|
|
|
|
int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm)
|
|
{
|
|
struct fca_init_spec *spec;
|
|
int ret;
|
|
unsigned long fca_ver, major, minor, detected_ver;
|
|
char x[3];
|
|
|
|
/* Make sure this is only run once */
|
|
if (mca_coll_fca_component.fca_context) {
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
fca_ver = FCA_API_CLEAR_MICRO(fca_get_version());
|
|
major = (fca_ver>>FCA_MAJOR_BIT);
|
|
minor = (fca_ver>>FCA_MINOR_BIT) & 0xf;
|
|
sprintf(x, "%ld%ld", major, minor);
|
|
detected_ver = atol(x);
|
|
|
|
FCA_VERBOSE(1, "FCA ABI version: %ld supported: %d", detected_ver, OMPI_FCA_VERSION);
|
|
|
|
if (detected_ver != OMPI_FCA_VERSION) {
|
|
FCA_ERROR("Unsupported FCA version: %s, please update FCA to v%d, now v%ld",
|
|
fca_get_version_string(),
|
|
OMPI_FCA_VERSION, fca_ver);
|
|
return OMPI_ERROR;
|
|
}
|
|
|
|
spec = fca_parse_spec_file(mca_coll_fca_component.fca_spec_file);
|
|
if (!spec) {
|
|
FCA_ERROR("Failed to parse FCA spec file `%s'", mca_coll_fca_component.fca_spec_file);
|
|
return OMPI_ERROR;
|
|
}
|
|
|
|
spec->job_id = ompi_proc_local()->proc_name.jobid;
|
|
spec->rank_id = ompi_comm_rank(MPI_COMM_WORLD);
|
|
spec->progress.func = mca_coll_fca_progress_cb;
|
|
spec->progress.arg = NULL;
|
|
ret = fca_init(spec, &mca_coll_fca_component.fca_context);
|
|
if (ret < 0) {
|
|
FCA_ERROR("Failed to initialize FCA: %s", fca_strerror(ret));
|
|
return OMPI_ERROR;
|
|
}
|
|
|
|
fca_free_init_spec(spec);
|
|
mca_coll_fca_init_fca_translations();
|
|
|
|
opal_progress_register(mca_coll_fca_mpi_progress_cb);
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static void mca_coll_fca_close_fca_lib(void)
|
|
{
|
|
opal_progress_unregister(mca_coll_fca_mpi_progress_cb);
|
|
if (mca_coll_fca_component.fca_context) {
|
|
fca_cleanup(mca_coll_fca_component.fca_context);
|
|
mca_coll_fca_component.fca_context = NULL;
|
|
}
|
|
}
|
|
|
|
static int fca_register(void)
|
|
{
|
|
mca_base_component_t *c;
|
|
|
|
FCA_VERBOSE(2, "==>");
|
|
|
|
c = &mca_coll_fca_component.super.collm_version;
|
|
|
|
mca_base_param_reg_int(c, "priority",
|
|
"Priority of the fca coll component",
|
|
false, false,
|
|
80,
|
|
&mca_coll_fca_component.fca_priority);
|
|
|
|
mca_base_param_reg_int(c, "verbose",
|
|
"Verbose level of the fca coll component",
|
|
false, false,
|
|
0,
|
|
&mca_coll_fca_component.fca_verbose);
|
|
|
|
mca_base_param_reg_int(c, "enable",
|
|
"[1|0|] Enable/Disable Fabric Collective Accelerator",
|
|
false, false,
|
|
1,
|
|
&mca_coll_fca_component.fca_enable);
|
|
|
|
mca_base_param_reg_string(c, "spec_file",
|
|
"Path to the FCA configuration file fca_mpi_spec.ini",
|
|
false, false,
|
|
""COLL_FCA_HOME"/etc/fca_mpi_spec.ini",
|
|
&mca_coll_fca_component.fca_spec_file);
|
|
|
|
mca_base_param_reg_int(c, "np",
|
|
"[integer] Minimal allowed job's NP to activate FCA",
|
|
false, false,
|
|
64,
|
|
&mca_coll_fca_component.fca_np);
|
|
|
|
mca_base_param_reg_int(c, "enable_barrier",
|
|
"[1|0|] Enable/Disable FCA Barrier support",
|
|
false, false,
|
|
OMPI_FCA_BCAST,
|
|
&mca_coll_fca_component.fca_enable_barrier);
|
|
|
|
mca_base_param_reg_int(c, "enable_bcast",
|
|
"[1|0|] Enable/Disable FCA Bcast support",
|
|
false, false,
|
|
OMPI_FCA_BCAST,
|
|
&mca_coll_fca_component.fca_enable_bcast);
|
|
|
|
mca_base_param_reg_int(c, "enable_reduce",
|
|
"[1|0|] Enable/Disable FCA Reduce support",
|
|
false, false,
|
|
OMPI_FCA_REDUCE,
|
|
&mca_coll_fca_component.fca_enable_reduce);
|
|
|
|
mca_base_param_reg_int(c, "enable_reduce_scatter",
|
|
"[1|0|] Enable/Disable FCA Reduce support",
|
|
false, false,
|
|
OMPI_FCA_REDUCE_SCATTER,
|
|
&mca_coll_fca_component.fca_enable_reduce_scatter);
|
|
|
|
mca_base_param_reg_int(c, "enable_allreduce",
|
|
"[1|0|] Enable/Disable FCA Allreduce support",
|
|
false, false,
|
|
OMPI_FCA_ALLREDUCE,
|
|
&mca_coll_fca_component.fca_enable_allreduce);
|
|
|
|
mca_base_param_reg_int(c, "enable_allgather",
|
|
"[1|0|] Enable/Disable FCA Allgather support",
|
|
false, false,
|
|
OMPI_FCA_ALLGATHER,
|
|
&mca_coll_fca_component.fca_enable_allgather);
|
|
|
|
mca_base_param_reg_int(c, "enable_allgatherv",
|
|
"[1|0|] Enable/Disable FCA Allgatherv support",
|
|
false, false,
|
|
OMPI_FCA_ALLGATHERV,
|
|
&mca_coll_fca_component.fca_enable_allgatherv);
|
|
|
|
mca_base_param_reg_int(c, "enable_gather",
|
|
"[1|0|] Enable/Disable FCA Gather support",
|
|
false, false,
|
|
OMPI_FCA_GATHER,
|
|
&mca_coll_fca_component.fca_enable_gather);
|
|
|
|
mca_base_param_reg_int(c, "enable_gatherv",
|
|
"[1|0|] Enable/Disable FCA Gatherv support",
|
|
false, false,
|
|
OMPI_FCA_GATHER,
|
|
&mca_coll_fca_component.fca_enable_gatherv);
|
|
|
|
|
|
mca_base_param_reg_int(c, "enable_alltoall",
|
|
"[1|0|] Enable/Disable FCA AlltoAll support",
|
|
false, false,
|
|
OMPI_FCA_ALLTOALL,
|
|
&mca_coll_fca_component.fca_enable_alltoall);
|
|
|
|
mca_base_param_reg_int(c, "enable_alltoallv",
|
|
"[1|0|] Enable/Disable FCA AlltoAllv support",
|
|
false, false,
|
|
OMPI_FCA_ALLTOALLV,
|
|
&mca_coll_fca_component.fca_enable_alltoallv);
|
|
|
|
mca_base_param_reg_int(c, "enable_alltoallw",
|
|
"[1|0|] Enable/Disable FCA AlltoAllw support",
|
|
false, false,
|
|
OMPI_FCA_ALLTOALLW,
|
|
&mca_coll_fca_component.fca_enable_alltoallw);
|
|
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int fca_open(void)
|
|
{
|
|
FCA_VERBOSE(2, "==>");
|
|
|
|
mca_coll_fca_output = opal_output_open(NULL);
|
|
opal_output_set_verbosity(mca_coll_fca_output, mca_coll_fca_component.fca_verbose);
|
|
|
|
mca_coll_fca_component.fca_context = NULL;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int fca_close(void)
|
|
{
|
|
FCA_VERBOSE(2, "==>");
|
|
|
|
if (!mca_coll_fca_component.fca_context)
|
|
return OMPI_SUCCESS;
|
|
|
|
mca_coll_fca_close_fca_lib();
|
|
return OMPI_SUCCESS;
|
|
}
|