2011-07-26 04:36:21 +00:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
|
|
|
|
|
* $COPYRIGHT$
|
|
|
|
|
*
|
|
|
|
|
* Additional copyrights may follow
|
|
|
|
|
*
|
|
|
|
|
* $HEADER$
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef MTL_MXM_H_HAS_BEEN_INCLUDED
|
|
|
|
|
#define MTL_MXM_H_HAS_BEEN_INCLUDED
|
|
|
|
|
|
|
|
|
|
#include <stdint.h>
|
2011-08-03 07:07:29 +00:00
|
|
|
#include <sys/types.h>
|
|
|
|
|
#include <unistd.h>
|
2011-07-26 04:36:21 +00:00
|
|
|
|
|
|
|
|
#include <mxm/api/mxm_api.h>
|
2013-04-24 16:51:43 +00:00
|
|
|
#ifndef MXM_VERSION
|
|
|
|
|
#define MXM_VERSION(major, minor) (((major)<<MXM_MAJOR_BIT)|((minor)<<MXM_MINOR_BIT))
|
|
|
|
|
#endif
|
2013-12-04 09:11:55 +00:00
|
|
|
|
|
|
|
|
#if MXM_API < MXM_VERSION(1,5)
|
|
|
|
|
#error "Unsupported MXM version, version 1.5 or above required"
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if MXM_API < MXM_VERSION(2,0)
|
2011-07-26 04:36:21 +00:00
|
|
|
#include <mxm/api/mxm_addr.h>
|
2013-04-12 16:37:42 +00:00
|
|
|
#endif
|
2011-07-26 04:36:21 +00:00
|
|
|
|
|
|
|
|
#include "ompi/mca/pml/pml.h"
|
|
|
|
|
#include "ompi/mca/mtl/mtl.h"
|
|
|
|
|
#include "ompi/mca/mtl/base/base.h"
|
George did the work and deserves all the credit for it. Ralph did the merge, and deserves whatever blame results from errors in it :-)
WHAT: Open our low-level communication infrastructure by moving all necessary components (btl/rcache/allocator/mpool) down in OPAL
All the components required for inter-process communications are currently deeply integrated in the OMPI layer. Several groups/institutions have express interest in having a more generic communication infrastructure, without all the OMPI layer dependencies. This communication layer should be made available at a different software level, available to all layers in the Open MPI software stack. As an example, our ORTE layer could replace the current OOB and instead use the BTL directly, gaining access to more reactive network interfaces than TCP. Similarly, external software libraries could take advantage of our highly optimized AM (active message) communication layer for their own purpose. UTK with support from Sandia, developped a version of Open MPI where the entire communication infrastucture has been moved down to OPAL (btl/rcache/allocator/mpool). Most of the moved components have been updated to match the new schema, with few exceptions (mainly BTLs where I have no way of compiling/testing them). Thus, the completion of this RFC is tied to being able to completing this move for all BTLs. For this we need help from the rest of the Open MPI community, especially those supporting some of the BTLs. A non-exhaustive list of BTLs that qualify here is: mx, portals4, scif, udapl, ugni, usnic.
This commit was SVN r32317.
2014-07-26 00:47:28 +00:00
|
|
|
#include "opal/class/ompi_free_list.h"
|
2012-07-25 13:26:40 +00:00
|
|
|
|
2013-02-17 09:51:03 +00:00
|
|
|
#include "opal/util/output.h"
|
|
|
|
|
#include "opal/util/show_help.h"
|
2011-07-26 04:36:21 +00:00
|
|
|
#include "opal/datatype/opal_convertor.h"
|
|
|
|
|
|
2011-08-09 10:54:08 +00:00
|
|
|
#include "mtl_mxm_debug.h"
|
|
|
|
|
|
2011-07-26 04:36:21 +00:00
|
|
|
BEGIN_C_DECLS
|
|
|
|
|
|
|
|
|
|
/* MTL interface functions */
|
|
|
|
|
extern int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t* mtl,
|
2013-08-30 16:54:55 +00:00
|
|
|
size_t nprocs, struct ompi_proc_t** procs);
|
2011-07-26 04:36:21 +00:00
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_del_procs(struct mca_mtl_base_module_t* mtl,
|
2013-08-30 16:54:55 +00:00
|
|
|
size_t nprocs, struct ompi_proc_t** procs);
|
2011-07-26 04:36:21 +00:00
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_send(struct mca_mtl_base_module_t* mtl,
|
|
|
|
|
struct ompi_communicator_t* comm, int dest, int tag,
|
|
|
|
|
struct opal_convertor_t *convertor,
|
|
|
|
|
mca_pml_base_send_mode_t mode);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_isend(struct mca_mtl_base_module_t* mtl,
|
|
|
|
|
struct ompi_communicator_t* comm, int dest,
|
|
|
|
|
int tag, struct opal_convertor_t *convertor,
|
|
|
|
|
mca_pml_base_send_mode_t mode, bool blocking,
|
|
|
|
|
mca_mtl_request_t * mtl_request);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_irecv(struct mca_mtl_base_module_t* mtl,
|
|
|
|
|
struct ompi_communicator_t *comm, int src,
|
|
|
|
|
int tag, struct opal_convertor_t *convertor,
|
|
|
|
|
struct mca_mtl_request_t *mtl_request);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_iprobe(struct mca_mtl_base_module_t* mtl,
|
|
|
|
|
struct ompi_communicator_t *comm, int src,
|
|
|
|
|
int tag, int *flag,
|
|
|
|
|
struct ompi_status_public_t *status);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_cancel(struct mca_mtl_base_module_t* mtl,
|
|
|
|
|
struct mca_mtl_request_t *mtl_request, int flag);
|
|
|
|
|
|
2012-03-26 19:27:03 +00:00
|
|
|
extern int ompi_mtl_mxm_imrecv(struct mca_mtl_base_module_t* mtl,
|
|
|
|
|
struct opal_convertor_t *convertor,
|
|
|
|
|
struct ompi_message_t **message,
|
|
|
|
|
struct mca_mtl_request_t *mtl_request);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_improbe(struct mca_mtl_base_module_t *mtl,
|
|
|
|
|
struct ompi_communicator_t *comm,
|
|
|
|
|
int src,
|
|
|
|
|
int tag,
|
|
|
|
|
int *matched,
|
|
|
|
|
struct ompi_message_t **message,
|
|
|
|
|
struct ompi_status_public_t *status);
|
|
|
|
|
|
2011-07-26 04:36:21 +00:00
|
|
|
extern int ompi_mtl_mxm_add_comm(struct mca_mtl_base_module_t *mtl,
|
|
|
|
|
struct ompi_communicator_t *comm);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_del_comm(struct mca_mtl_base_module_t *mtl,
|
|
|
|
|
struct ompi_communicator_t *comm);
|
|
|
|
|
|
|
|
|
|
extern int ompi_mtl_mxm_finalize(struct mca_mtl_base_module_t* mtl);
|
|
|
|
|
|
|
|
|
|
int ompi_mtl_mxm_module_init(void);
|
|
|
|
|
|
2012-07-25 13:26:40 +00:00
|
|
|
struct ompi_mtl_mxm_message_t {
|
|
|
|
|
ompi_free_list_item_t super;
|
|
|
|
|
|
|
|
|
|
mxm_mq_h mq;
|
|
|
|
|
mxm_conn_h conn;
|
|
|
|
|
mxm_message_h mxm_msg;
|
|
|
|
|
|
|
|
|
|
mxm_tag_t tag;
|
|
|
|
|
mxm_tag_t tag_mask;
|
|
|
|
|
};
|
|
|
|
|
typedef struct ompi_mtl_mxm_message_t ompi_mtl_mxm_message_t;
|
|
|
|
|
OBJ_CLASS_DECLARATION(ompi_mtl_mxm_message_t);
|
|
|
|
|
|
2011-07-26 04:36:21 +00:00
|
|
|
END_C_DECLS
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|