openmpi/ompi/mca/ptl/gm/ptl_gm.h

/* -*- Mode: C; c-basic-offset:4 ; -*- */

/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University.
 *                         All rights reserved.
 * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
 *                         All rights reserved.
 * Copyright (c) 2004 The Ohio State University.
 *                    All rights reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */
/**
 * @file
 */
#ifndef MCA_PTL_GM_H
#define MCA_PTL_GM_H

#include "ompi_config.h"
#include "class/ompi_free_list.h"
#include "mca/ptl/ptl.h"
#include "ptl_gm_priv.h"
#include "ptl_gm_peer.h"

#define MCA_PTL_GM_STATISTICS 0
#define MAX_RECV_TOKENS 256
#define PTL_GM_ADMIN_SEND_TOKENS 0
#define PTL_GM_ADMIN_RECV_TOKENS 0
#define GM_SIZE  30
#define NUM_RECV_FRAGS 256 
#define MCA_PTL_GM_FRAG_CACHED
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif

    /**
     * GM PTL component
     */
    struct mca_ptl_gm_component_t {
        mca_ptl_base_component_1_0_0_t super; /**< base PTL module */
        struct mca_ptl_gm_module_t **gm_ptl_modules; /**< array of available PTL modules */
        size_t      gm_num_ptl_modules;      /**< number of ptls actually used */
        size_t      gm_max_ptl_modules;      /**< maximum number of ptls - available */
        uint32_t    gm_max_port_number;      /**< maximum number of ports by board */
        uint32_t    gm_max_boards_number;    /**< maximum number of boards on the node */
        uint32_t    gm_free_list_num;        /**< initial size of free lists */
        uint32_t    gm_free_list_max;        /**< maximum size of free lists */
        uint32_t    gm_free_list_inc;        /**< number of elements to alloc when growing free lists */
        uint32_t    gm_segment_size;         /**< size of the allocated segment */
        uint32_t    gm_eager_limit;          /**< number of bytes before the rendez-vous protocol. If the
                                              **< size of the message is less than this number then GM 
                                              **< use a eager protocol.  */
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET
        uint32_t    gm_rndv_burst_limit;
        uint32_t    gm_rdma_frag_size;       /**< maximum fragment size used to transfer data over RDMA */
#endif  /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
        char*       gm_port_name;            /**< the name used to get the port */

        struct mca_ptl_gm_proc_t* gm_local;
        opal_list_t gm_procs;
        opal_list_t gm_send_req;
        ompi_free_list_t gm_unexpected_frags_data;

        opal_mutex_t gm_lock;                 /**< lock for accessing module state */
    };

    typedef struct mca_ptl_gm_component_t mca_ptl_gm_component_t;
    extern mca_ptl_gm_component_t mca_ptl_gm_component;

    /**
     * GM PTL Interface
     */
    struct mca_ptl_gm_module_t {
        mca_ptl_base_module_t super;    /**< base PTL module interface */
        struct gm_port *gm_port;
        mca_ptl_gm_addr_t local_addr;
        unsigned int num_send_tokens;
        unsigned int num_recv_tokens;
        unsigned int max_send_tokens;
        unsigned int max_recv_tokens;
        void*        gm_send_dma_memory; /**< pointer to the send DMA registered memory attached to the PTL */
        void*        gm_recv_dma_memory; /**< pointer to the recv DMA registered memory attached to the PTL */
        struct mca_ptl_gm_send_frag_t* gm_send_fragments;
        struct mca_ptl_gm_recv_frag_t* gm_recv_fragments;

        ompi_free_list_t gm_send_frags;
        ompi_free_list_t gm_send_dma_frags;
        ompi_free_list_t gm_recv_frags_free;
        opal_list_t gm_send_frags_queue;
        opal_list_t gm_pending_acks;
        opal_list_t gm_recv_outstanding_queue;

        opal_thread_t thread;
#if MCA_PTL_GM_STATISTICS
        size_t      ptl_bytes_sent;
        size_t      ptl_bytes_recv;
#endif  /* MCA_PTL_GM_STATISTICS */
    };

    typedef struct mca_ptl_gm_module_t mca_ptl_gm_module_t;
    extern mca_ptl_gm_module_t mca_ptl_gm_module;

    /**
     * Register GM module parameters with the MCA framework
     */
    extern int  mca_ptl_gm_component_open (void);

    /**
     * Any final cleanup before being unloaded.
     */
    extern int  mca_ptl_gm_component_close (void);

    /**
     * GM module initialization.
     * 
     * @param num_ptls (OUT)                  Number of PTLs returned in PTL array.
     * @param allow_multi_user_threads (OUT)  Flag indicating wether PTL supports user threads (TRUE)
     * @param have_hidden_threads (OUT)       Flag indicating wether PTL uses threads (TRUE)
     */
    extern mca_ptl_base_module_t **mca_ptl_gm_component_init (int *num_ptl_modules,
                                                              bool enable_progress_threads,
                                                              bool enable_mpi_threads);

    /**
     * GM module control.
     */
    extern int  mca_ptl_gm_component_control (int param, void *value, size_t size);

    /**
     * GM module progress.
     */
    extern int  mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp);

    /**
     *  GM put
     */
    extern int  mca_ptl_gm_put( struct mca_ptl_base_module_t *ptl,
                                struct mca_ptl_base_peer_t *ptl_peer,
                                struct mca_ptl_base_send_request_t *sendreq,
                                size_t offset, size_t size, int flags);

    /**
     *  GM get
     */
    extern int  mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl,
                                struct mca_ptl_base_peer_t *ptl_peer,
                                struct mca_ptl_base_recv_request_t *sendreq,
                                size_t offset, size_t size, int flags);

    /**
     * PML->PTL notification of change in the process list.
     * 
     * @param ptl (IN)
     * @param nprocs (IN)     Number of processes
     * @param procs (IN)      Set of processes
     * @param peers (OUT)     Set of (optional) peer addressing info.
     * @param peers (IN/OUT)  Set of processes that are reachable via this PTL.
     * @return     OMPI_SUCCESS or error status on failure.
     * 
     */
    extern int  mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
                                      size_t nprocs,
                                      struct ompi_proc_t **procs,
                                      struct mca_ptl_base_peer_t **peers,
                                      struct ompi_bitmap_t * reachable);

    /**
     * PML->PTL notification of change in the process list.
     *
     * @param ptl (IN)     PTL instance
     * @param nproc (IN)   Number of processes.
     * @param procs (IN)   Set of processes.
     * @param peers (IN)   Set of peer data structures.
     * @return             Status indicating if cleanup was successful
     *
     */
    extern int  mca_ptl_gm_del_procs( struct mca_ptl_base_module_t *ptl,
                                      size_t nprocs,
                                      struct ompi_proc_t **procs,
                                      struct mca_ptl_base_peer_t **peers );

    /**
     * PML->PTL Allocate a send request from the PTL modules free list.
     *
     * @param ptl (IN)       PTL instance
     * @param request (OUT)  Pointer to allocated request.
     * @return               Status indicating if allocation was successful.
     *
     */
    extern int  mca_ptl_gm_request_init( struct mca_ptl_base_module_t* ptl,
                                         struct mca_ptl_base_send_request_t* req);

    /**
     *
     */
    extern void mca_ptl_gm_request_fini( struct mca_ptl_base_module_t *ptl,
                                         struct mca_ptl_base_send_request_t* req);

    /**
     * PML->PTL Notification that a receive fragment has been matched.
     *
     * @param ptl (IN)          PTL instance
     * @param recv_frag (IN)    Receive fragment
     *
     */
    extern void mca_ptl_gm_matched (struct mca_ptl_base_module_t *ptl,
                                    struct mca_ptl_base_recv_frag_t *frag);

    /**
     *
     */
    extern int  mca_ptl_gm_finalize (struct mca_ptl_base_module_t *ptl);

    /**
     * Internally allocate memory for the unexpected messages. We will manage a list
     * of such buffers in order to avoid too many memory allocations.
     */
    extern char* gm_get_local_buffer( void );
    extern void gm_release_local_buffer( char* ptr );

    union mca_ptl_base_header_t;
    void mca_ptl_gm_dump_header( char* str, union mca_ptl_base_header_t* hdr );

#if OMPI_ENABLE_DEBUG
#include "opal/class/opal_list.h"
/* If debug is enabled we have to work around the item validity checks. */
#define OMPI_GM_FREE_LIST_RETURN( LIST, ITEM )    \
do {                                              \
    (ITEM)->opal_list_item_refcount = 0;          \
    (ITEM)->opal_list_item_belong_to = NULL;      \
    (ITEM)->super.cls_init_file_name = __FILE__;  \
    (ITEM)->super.cls_init_lineno = __LINE__;     \
    OMPI_FREE_LIST_RETURN( (LIST), (ITEM) );      \
} while(0)
#else
#define OMPI_GM_FREE_LIST_RETURN OMPI_FREE_LIST_RETURN
#endif  /* OMPI_ENABLE_DEBUG */

#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif