232 строки
6.1 KiB
C
232 строки
6.1 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*
|
|
*/
|
|
|
|
#if !defined(MCA_OOB_UD_REQ_H)
|
|
#define MCA_OOB_UD_REQ_H
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/types.h"
|
|
#include "opal/threads/condition.h"
|
|
#include "opal/mca/event/event.h"
|
|
#include "orte/mca/rml/rml.h"
|
|
|
|
#include <infiniband/verbs.h>
|
|
|
|
#include "oob_ud_qp.h"
|
|
|
|
struct mca_oob_ud_peer_t;
|
|
|
|
enum mca_oob_ud_req_type_t {
|
|
MCA_OOB_UD_REQ_RECV,
|
|
MCA_OOB_UD_REQ_SEND,
|
|
MCA_OOB_UD_REQ_UNEX
|
|
};
|
|
typedef enum mca_oob_ud_req_type_t mca_oob_ud_req_type_t;
|
|
|
|
enum mca_oob_ud_req_state_t {
|
|
MCA_OOB_UD_REQ_ACTIVE,
|
|
MCA_OOB_UD_REQ_PENDING,
|
|
MCA_OOB_UD_REQ_COMPLETE
|
|
};
|
|
|
|
typedef enum mca_oob_ud_req_state_t mca_oob_ud_req_state_t;
|
|
|
|
struct mca_oob_ud_req_t {
|
|
opal_list_item_t super;
|
|
|
|
mca_oob_ud_req_type_t type;
|
|
mca_oob_ud_req_state_t state;
|
|
|
|
union {
|
|
struct ibv_send_wr *send;
|
|
struct ibv_recv_wr *recv;
|
|
} req_wr;
|
|
|
|
/* storage for ib grh */
|
|
struct ibv_grh *req_grh;
|
|
struct ibv_mr *req_grh_mr;
|
|
|
|
/* memory register for iovec memory */
|
|
struct ibv_mr **req_mr;
|
|
|
|
struct ibv_sge *req_sge;
|
|
|
|
/* negotiated mtu */
|
|
int req_mtu;
|
|
uint32_t req_rem_qpn;
|
|
int req_rem_data_len;
|
|
|
|
int req_packet_count;
|
|
|
|
struct mca_oob_ud_peer_t *req_peer;
|
|
struct mca_oob_ud_port_t *req_port;
|
|
struct mca_oob_ud_qp_t *req_qp;
|
|
|
|
/* remote context (request or response) */
|
|
void *req_rem_ctx;
|
|
|
|
/* retry timer */
|
|
struct {
|
|
opal_event_t event;
|
|
struct timeval value;
|
|
} timer;
|
|
|
|
/* user request */
|
|
orte_process_name_t req_target;
|
|
orte_process_name_t req_origin;
|
|
struct iovec *req_uiov;
|
|
int req_count;
|
|
int req_tag;
|
|
int req_flags;
|
|
int req_rc;
|
|
|
|
orte_rml_callback_fn_t req_cbfunc;
|
|
void *req_cbdata;
|
|
|
|
/* what list is this request in */
|
|
opal_list_t *req_list;
|
|
|
|
bool req_is_eager;
|
|
};
|
|
typedef struct mca_oob_ud_req_t mca_oob_ud_req_t;
|
|
OBJ_CLASS_DECLARATION(mca_oob_ud_req_t);
|
|
|
|
enum mca_oob_ud_status_t {
|
|
/* message posted */
|
|
MCA_OOB_UD_MSG_STATUS_POSTED,
|
|
/* remote side receive the message (ack'd) */
|
|
MCA_OOB_UD_MSG_STATUS_COMPLETE,
|
|
/* request message timed out */
|
|
MCA_OOB_UD_MSG_STATUS_TIMEOUT,
|
|
/* other failure */
|
|
MCA_OOB_UD_MSG_STATUS_ERROR
|
|
};
|
|
typedef enum mca_oob_ud_status_t mca_oob_ud_status_t;
|
|
|
|
enum mca_oob_ud_msg_type_t {
|
|
MCA_OOB_UD_MSG_REQUEST = 37,
|
|
MCA_OOB_UD_MSG_REPLY = 38,
|
|
MCA_OOB_UD_MSG_COMPLETE = 39,
|
|
MCA_OOB_UD_MSG_PING = 40,
|
|
MCA_OOB_UD_MSG_ACK = 41,
|
|
MCA_OOB_UD_MSG_NACK = 42,
|
|
MCA_OOB_UD_MSG_DATA_OK = 43,
|
|
MCA_OOB_UD_MSG_END = 44
|
|
};
|
|
typedef enum mca_oob_ud_msg_type_t mca_oob_ud_msg_type_t;
|
|
|
|
struct mca_oob_ud_msg_hdr_t {
|
|
mca_oob_ud_msg_type_t msg_type;
|
|
|
|
void *msg_rem_ctx;
|
|
void *msg_lcl_ctx;
|
|
|
|
orte_process_name_t msg_origin;
|
|
|
|
uint64_t msg_id;
|
|
|
|
struct {
|
|
/* the receiver can get the qpn and lid from the work completion */
|
|
uint32_t qkey;
|
|
orte_process_name_t name;
|
|
uint8_t port_num;
|
|
} ra;
|
|
|
|
union {
|
|
struct {
|
|
int tag;
|
|
int data_len;
|
|
int mtu;
|
|
bool data_follows;
|
|
} req;
|
|
struct {
|
|
uint32_t qpn;
|
|
int data_len;
|
|
int tag;
|
|
int mtu;
|
|
} rep;
|
|
} msg_data;
|
|
};
|
|
typedef struct mca_oob_ud_msg_hdr_t mca_oob_ud_msg_hdr_t;
|
|
|
|
struct mca_oob_ud_msg_t {
|
|
opal_free_list_item_t super;
|
|
|
|
struct ibv_send_wr wr;
|
|
struct ibv_sge sge;
|
|
mca_oob_ud_msg_hdr_t *hdr;
|
|
struct ibv_mr *mr;
|
|
|
|
/* qp this request was sent over */
|
|
struct mca_oob_ud_qp_t *qp;
|
|
struct mca_oob_ud_port_t *port;
|
|
|
|
opal_mutex_t lock;
|
|
opal_condition_t status_changed;
|
|
mca_oob_ud_status_t status;
|
|
|
|
bool persist;
|
|
mca_oob_ud_req_t *req;
|
|
|
|
void (*cbfunc) (struct mca_oob_ud_msg_t *, int);
|
|
|
|
struct mca_oob_ud_peer_t *peer;
|
|
};
|
|
typedef struct mca_oob_ud_msg_t mca_oob_ud_msg_t;
|
|
OBJ_CLASS_DECLARATION(mca_oob_ud_msg_t);
|
|
|
|
static inline int mca_oob_ud_recv_alloc (mca_oob_ud_req_t *recv_req)
|
|
{
|
|
int iov_index;
|
|
|
|
if (recv_req->req_flags & ORTE_RML_ALLOC) {
|
|
size_t alloc_size = recv_req->req_rem_data_len;
|
|
|
|
for (iov_index = 0 ; iov_index < recv_req->req_count - 1 ; ++iov_index) {
|
|
alloc_size -= recv_req->req_uiov[iov_index].iov_len;
|
|
}
|
|
|
|
recv_req->req_uiov[iov_index].iov_len = alloc_size;
|
|
recv_req->req_uiov[iov_index].iov_base = calloc (alloc_size, 1);
|
|
|
|
if (NULL == recv_req->req_uiov[iov_index].iov_base) {
|
|
return ORTE_ERROR;
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
int mca_oob_ud_msg_get (struct mca_oob_ud_port_t *port, mca_oob_ud_req_t *req,
|
|
mca_oob_ud_qp_t *qp, mca_oob_ud_peer_t *peer, bool persist,
|
|
mca_oob_ud_msg_t **msgp);
|
|
int mca_oob_ud_msg_init (mca_oob_ud_msg_t *msg, struct mca_oob_ud_port_t *port,
|
|
char *buf, struct ibv_mr *mr);
|
|
void mca_oob_ud_msg_return (mca_oob_ud_msg_t *msg);
|
|
|
|
|
|
void mca_oob_ud_req_timer_set (mca_oob_ud_req_t *req, const struct timeval *timeout,
|
|
int max_tries, void (*cb)(evutil_socket_t, short, void *));
|
|
|
|
int mca_oob_ud_msg_post_send (mca_oob_ud_msg_t *msg);
|
|
int mca_oob_ud_msg_wait (mca_oob_ud_msg_t *msg);
|
|
|
|
int mca_oob_ud_msg_status_update (mca_oob_ud_msg_t *msg, mca_oob_ud_status_t status);
|
|
|
|
void mca_oob_ud_req_complete (mca_oob_ud_req_t *req, int rc);
|
|
void mca_oob_ud_req_abort (mca_oob_ud_req_t *req);
|
|
|
|
void mca_oob_ud_req_append_to_list (mca_oob_ud_req_t *req, opal_list_t *list);
|
|
bool mca_oob_ud_req_is_in_list (mca_oob_ud_req_t *req, opal_list_t *list);
|
|
|
|
#endif
|