openmpi/ompi/mca/bcol/basesmuma/bcol_basesmuma.h

/*
 * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
 * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
 * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */


#ifndef MCA_BCOL_basesmuma_EXPORT_H
#define MCA_BCOL_basesmuma_EXPORT_H

#include "ompi_config.h"
#include "ompi/mca/bcol/bcol.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/mca/coll/ml/coll_ml.h"
#include "ompi/mca/coll/ml/coll_ml_allocation.h"
#include "ompi/request/request.h"
#include "ompi/proc/proc.h"
#include "ompi/patterns/net/netpatterns.h"

#include "opal/mca/mca.h"
#include "opal/util/arch.h"
#include "opal/util/argv.h"
#include "opal/datatype/opal_datatype.h"
#include "opal/util/output.h"

#include "bcol_basesmuma_smcm.h"
BEGIN_C_DECLS

struct list_data_t  {
    opal_list_item_t super;
    void *data;
};
typedef struct list_data_t list_data_t;
OBJ_CLASS_DECLARATION(list_data_t);

/*
 * Macro's for manipulating the 64 bit shared memory control bits.
 * The 64 bit field is devided into 4 bit fields
 *
 *   | 48-63: src  |  32-47: index |  16-31: flag |  0-15: sequence number |
 *
 * Only the low 16 bits of the sequence number will be put in the header
 * space.  We will use the fact that the use of the shared buffers is
 * synchronous, and get the upper 48 bits from the local process space.
 */

#define BASESMUMA_CACHE_LINE_SIZE 128

#define SHIFT_UP   <<
#define SHIFT_DOWN >>

#define SEQ_WIDTH  16
#define SEQ_BASE    0
#define FIELD_SEQ_MASK   ( ( 1 SHIFT_UP SEQ_WIDTH ) - 1 )
#define INPLACE_SEQ_MASK ( (int64_t)FIELD_SEQ_MASK SHIFT_UP SEQ_BASE)

#define FLAG_WIDTH 16
#define FLAG_BASE  16
#define FIELD_FLAG_MASK   ( ( 1 SHIFT_UP FLAG_WIDTH ) - 1 )
#define INPLACE_FLAG_MASK ( (int64_t)FIELD_FLAG_MASK SHIFT_UP FLAG_BASE)

#define INDX_WIDTH 16
#define INDX_BASE  32
#define FIELD_INDX_MASK   ( ( 1 SHIFT_UP INDX_WIDTH ) - 1 )
#define INPLACE_INDX_MASK ( (int64_t)FIELD_INDX_MASK SHIFT_UP INDX_BASE)

#define SRC_WIDTH  16
#define SRC_BASE   48
#define FIELD_SRC_MASK   ( ( 1 SHIFT_UP SRC_WIDTH ) - 1 )
#define INPLACE_SRC_MASK ( (int64_t)FIELD_SRC_MASK SHIFT_UP SRC_BASE)
/*int64_t INPLACE_SRC_MASK= ((int64_t)FIELD_SRC_MASK SHIFT_UP SRC_BASE); */


#define EXTRACT_FLAG(INPUT, OUTPUT, OUTPUT_TYPE, FIELD_BASE, FIELD_MASK) \
        OUTPUT = (OUTPUT_TYPE) ( (INPUT SHIFT_DOWN FIELD_BASE ) & FIELD_MASK )

#define STORE_FLAG(INPUT, OUTPUT, INPUT_TYPE, OUTPUT_TYPE, FIELD_BASE, INPLACE_FIELD_MASK ) \
        OUTPUT =  \
     ( \
         /* 3 */ \
         ( \
             /* 2 */ \
             ( \
                 /* 1 - shift the input field to the proper location */ \
                 (OUTPUT_TYPE)( \
                   ((OUTPUT_TYPE)((INPUT_TYPE) (INPUT)))  \
                        SHIFT_UP FIELD_BASE ) \
                 /* mask off the extra bits */ \
                  & ((OUTPUT_TYPE)INPLACE_FIELD_MASK)  \
             ) \
         /* store back to the OUTPUT field, w/o destroying other fields */ \
         ) | OUTPUT \
     )

/** 
 * Structure to hold the basic shared memory bcoll component.  
 */
struct mca_bcol_basesmuma_component_t {
    /** Base coll component */
    mca_bcol_base_component_2_0_0_t super;

    /* management data for collectives with no user data */

    /** MCA parameter: number of memory banks */
    int basesmuma_num_mem_banks;

    /** MCA parameter: number of regions per memory bank */
    int basesmuma_num_regions_per_bank;

    /** MCA parameter: Number of simultaneous groups supported */
    int n_groups_supported;

    /* management data for collectives with user data (ud) - the memory
     * is actually obtained at the ML level
     */

    /** MCA paramenter:  number of polling loops to run while waiting
     *  for children or parent to complete their work
     */
    int n_poll_loops;

    /* mpool size */
    size_t mpool_size;


    /* mpool inited - will use this to test whether or not the 
     * shared memory has been inited 
     */
    bool mpool_inited;

    /* shared memory control buffer - the control structures reside
     *   in shared memory */
    bcol_basesmuma_smcm_mmap_t *sm_ctl_structs;

    /* shared memory payload buffer 
     */
    bcol_basesmuma_smcm_mmap_t *sm_payload_structs;

    /*
     * list of shared memory control structures
     */
    opal_list_t ctl_structures;


    /** opal list in which the list of peers that I am "connected" to is stored 
     */
    opal_list_t sm_connections_list;

    /* opal list in which the list of payload peers that I am "connected" to
     * is stored
     */
    opal_list_t sm_payload_connections_list; 	

    /*
     * list of non-blocking admin barriers to progress */
    opal_mutex_t nb_admin_barriers_mutex;
    opal_list_t nb_admin_barriers;

    /*
     * order of fan-in tree
     */
    int radix_fanin;

    /*
     * order of fan-out tree
     */
    int radix_fanout;

    /*
     * Order of read tree 
     */
    int radix_read_tree;

    /*
     * order of reduction fan-out tree
     */
    int order_reduction_tree;

    /*
     * K-nomial tree radix
     */
    int k_nomial_radix;

    /*
     * K-ary scatter tree radix
     */
    int scatter_kary_radix;

    /*
     * number of polling loops 
     */
    int num_to_probe;

	/*
	 * Portals addressing info
	 * void*: because wanted to keep portal library dependencies 
	 * as local as possible
	 */
	void *portals_info;
	bool portals_init;

    /*
     * verbosity level
     */
    int verbose;

    /*
     * control file name base string
     */
    char *clt_base_fname;

    /*
     * data file name base string
     */
    char *payload_base_fname;

    /*
     * shared memory scratch space.  This is mapped at the end of the
     * segement of memory holding the control structures.
     */
    char *my_scratch_shared_memory;

    /*
     * size of scratch memory
     */
    size_t my_scratch_shared_memory_size;

    /* the offset will be the same for all ranks */
    size_t scratch_offset_from_base_ctl_file;
};

static inline int mca_bcol_basesmuma_err(const char* fmt, ...)
{
    va_list list;
    int ret;

    va_start(list, fmt);
    ret = vfprintf(stderr, fmt, list);
    va_end(list);
    return ret;
}

#if OPAL_ENABLE_DEBUG
#define BASESMUMA_VERBOSE(level, args)                              \
do {                                                     \
    if(mca_bcol_basesmuma_component.verbose >= level) {         \
        mca_bcol_basesmuma_err("[%s]%s[%s:%d:%s] BCOL-BASESMUMA ",     \
                ompi_process_info.nodename,              \
                OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),      \
                __FILE__, __LINE__, __func__);           \
        mca_bcol_basesmuma_err args;                            \
        mca_bcol_basesmuma_err("\n");                           \
    }                                                    \
} while(0)
#else
#define BASESMUMA_VERBOSE(level, args)
#endif


/** 
 * Convenience typedef */
typedef struct mca_bcol_basesmuma_component_t mca_bcol_basesmuma_component_t;

#if 0
    /* 
     * Implemented function index list 
     */

    /* barrier */
    enum{
        FANIN_FAN_OUT_BARRIER_FN,
        RECURSIVE_DOUBLING_BARRIER_FN,
        N_BARRIER_FNS
    };

    /* reduce */
    enum{
        FANIN_REDUCE_FN,
        REDUCE_SCATTER_GATHER_FN,
        N_REDUCE_FNS
    };
    enum{
        SHORT_DATA_FN_REDUCE,
        LONG_DATA_FN_REDUCE,
        N_REDUCE_FNS_USED
    };

    /* all-reduce */
    enum{
        FANIN_FANOUT_ALLREDUCE_FN,
        REDUCE_SCATTER_ALLGATHER_FN,
        N_ALLREDUCE_FNS
    };
    enum{
        SHORT_DATA_FN_ALLREDUCE,
        LONG_DATA_FN_ALLREDUCE,
        N_ALLREDUCE_FNS_USED
    };


    /* enum for node type */
    enum{
        ROOT_NODE,
        LEAF_NODE,
        INTERIOR_NODE
    };


    /*
     * N-order tree node description
     */
    struct tree_node_t {
        /* my rank within the group */
        int my_rank;
        /* my node type - root, leaf, or interior */
        int my_node_type;
        /* number of nodes in the tree */
        int tree_size;
        /* number of parents (0/1) */
        int n_parents;
        /* number of children */
        int n_children;
        /* parent rank within the group */
        int parent_rank;
        /* chidren ranks within the group */
        int *children_ranks;
    };
    typedef struct tree_node_t tree_node_t;

    /*
     * Pair-wise data exchange
     */
    /* enum for node type */
    enum{
        EXCHANGE_NODE,
        EXTRA_NODE
    };

    struct pair_exchange_node_t {

	/* my rank within the group */
	int my_rank;

        /* number of nodes this node will exchange data with */
        int n_exchanges;

        /* ranks of nodes involved in data exchnge */
        int *rank_exchanges;

        /* number of extra sources of data - outside largest power of 2 in
         *  this group */
        int n_extra_sources;
        
        /* rank of the extra source */
        int rank_extra_source;

        /* number of tags needed per stripe */
        int n_tags;

        /* log 2 of largest full power of 2 for this node set */
        int log_2;

        /* largest power of 2 that fits in this group */
        int n_largest_pow_2;

        /* node type */
        int node_type;

    };
    typedef struct pair_exchange_node_t pair_exchange_node_t;
#endif
    /*
     * descriptor for managing the admin nonblocking barrier routine.
     *   This is an sm internal routine, and assumes only 1 outstanding
     *   nb-barrier collective call per block.
     */
    /* forward declarations */
    struct mca_bcol_basesmuma_module_t;
    struct sm_buffer_mgmt;

    struct sm_nbbar_desc_t {
        /* make sure we can put this on a list */
        opal_list_item_t super;

        /* phase of the collective operation - needed to know how to continue 
         * progressing the nb-barrier */
        int collective_phase;

        /* iteration to continue at */
        int recursive_dbl_iteration;

        /* pointer to the collective module this is associated with */
        struct mca_bcol_basesmuma_module_t *sm_module;

	    /* pointer to payload/control structs buffers */
        struct sm_buffer_mgmt *coll_buff;

        /* pool index */
        int pool_index;

        /* pointer to the ml_memory_block_desc_t structure
         * that is actually managing this registration.
         * This is meaningful when these control structures
         * are used in conjunction with the user payload
         * data that is allocated at the ml level.
         */
         void *ml_memory_block_descriptor;

    };
    typedef struct sm_nbbar_desc_t sm_nbbar_desc_t;

    /*
     * Barrier request objects
     */

    /* shared memory data strucutures */
    struct mca_bcol_basesmuma_nb_request_process_shared_mem_t {
        volatile uint64_t coll_index;
        /* flag used to indicate the status of this memory region */
        volatile uint64_t flag;
        volatile uint64_t index;

        /* pading */
        /* Note: need to change this so it takes less memory */
        char padding[BASESMUMA_CACHE_LINE_SIZE-3*sizeof(uint64_t)];
    };

    typedef struct mca_bcol_basesmuma_nb_request_process_shared_mem_t
        mca_bcol_basesmuma_nb_request_process_shared_mem_t;

    /* enum for phase at which the nb barrier is in */
    enum{
        NB_BARRIER_INACTIVE,

        /* fan-in/fan-out */
        NB_BARRIER_FAN_IN,
        NB_BARRIER_FAN_OUT,

        /* recursive doubling */
        NB_PRE_PHASE,
        NB_RECURSIVE_DOUBLING,
        NB_POST_PHASE,

        /* done and not started are the same for all practicle
         * purposes, as the init funtion always sets this flag
         */
        NB_BARRIER_DONE
    };



    /* forward declartion */
    struct mca_bcol_basesmuma_module_t;


    /* control segment for shared memory */
    struct mca_bcol_basesmuma_ctl_struct_t {
	    /* collective identifier */
	    volatile int64_t sequence_number;
	    volatile int64_t flag;
        volatile int64_t index;
        volatile int64_t offset;
        volatile int64_t offset_zip;
        

        /* used for non-blocking algorithms */
        int status;
        int active_requests;
        int iteration;

        int *src_ptr;
        
        int start;

        /* process private data */
        int starting_flag_value;

        /* experiment for large data colls */
        int n_sends;
        int length;


#ifdef __PORTALS_AVAIL__
		struct mca_bcol_basesmuma_portal_buf_addr_t portals_buf_addr;	
#endif
	    /* padding */
        /* ok, no room to pad anymore */
	    /*char padding[BASESMUMA_CACHE_LINE_SIZE-5*sizeof(int64_t)-8*sizeof(int)];*/
    };
    typedef struct mca_bcol_basesmuma_ctl_struct_t mca_bcol_basesmuma_ctl_struct_t;


#define SM_BCOLS_MAX 2

    enum {
        ALLGATHER_FLAG,
        ALLREDUCE_FLAG,
        BARRIER_FANIN_FLAG,
        BARRIER_FANOUT_FLAG,
        BARRIER_RKING_FLAG,
        BCAST_FLAG,
        GATHER_FLAG,
        REDUCE_FLAG,
        NUM_SIGNAL_FLAGS
    };

    
    /* control region for colls with user data - shared memory */
    struct mca_bcol_basesmuma_header_t {
        /* collective identifier */
        volatile int64_t sequence_number;
        volatile int8_t  flags[NUM_SIGNAL_FLAGS][SM_BCOLS_MAX];
        volatile int32_t src; /* src of bcast data for unknown root, 
                                 bcol id for known root
                               */
        /* starting flag - hierarchies */
        int8_t starting_flag_value[SM_BCOLS_MAX];

    };
    typedef struct mca_bcol_basesmuma_header_t mca_bcol_basesmuma_header_t;

    /* data needed for large messages */
    struct mca_bcol_basesmuma_large_msg_t {
        /* scatter allgather data */
        uint64_t offset;
        uint64_t n_sends;
        uint64_t length;

        /* portals data */

    };
    typedef struct mca_bcol_basesmuma_large_msg_t mca_bcol_basesmuma_large_msg_t;

    /* payload struct */
	struct mca_bcol_basesmuma_payload_t {

		/* base pointer to shared memory control structure */
		mca_bcol_basesmuma_header_t *ctl_struct;
        void *payload;
        
	};

	typedef struct mca_bcol_basesmuma_payload_t mca_bcol_basesmuma_payload_t;




    /* memory bank memory management structure */
    struct mem_bank_management_t {

        /* generation counter */
        uint64_t bank_gen_counter;

        /* descriptor for the non-blocking barrier.  This is
         *  used to manage this bank of memory.
         */
        sm_nbbar_desc_t nb_barrier_desc;

        /* the number of buffers that are not in use, and are
         * available.  The assumption is that the buffers are
         * recycled all at once, so are available for re-use
         * until all buffers have been made available for re-use.
         */
        volatile int available_buffers;

        /*
         * number of buffers freed */
        volatile int n_buffs_freed;

        /* mutex to ensure atomic recycling of resrouces */
        opal_mutex_t mutex;

        /* number of buffers being managed */
        int number_of_buffers;

        /* shared memory control structures */
        int index_shared_mem_ctl_structs;


    };
    typedef struct mem_bank_management_t mem_bank_management_t;

/* data structure for shared buffers */
struct sm_buffer_mgmt {
    /* number of buffers per process */
    int number_of_buffs;

    /* size of group */
    int size_of_group;

    /* number of memory banks */
    int num_mem_banks;

    /* number of buffers per memory bank */
    int num_buffs_per_mem_bank;

    /* log base 2 of num_buffs_per_mem_bank */
    int log2_num_buffs_per_mem_bank;

    /* log base 2 total number of buffers */
    int log2_number_of_buffs;

    /* mask - masks off the bits corresponding to buffer index */
    int mask;

    /* control buffers - these point to regions in shared memory */
    /* leading dimension is the group size - all pointers for a given
     * set of buffers appear consecutively in this array
     */
    volatile void **ctl_buffs;

    /* management data for the control structures - 
     * one per bank of control structures - Will be used for 
     * the payload buffers as well.
     */
    mem_bank_management_t *ctl_buffs_mgmt;

    /* data buffers - these point to regions in shared memory */
    /* leading dimension is the group size - all pointers for a given
     * set of buffers appear consecutively in this array
     */

    volatile mca_bcol_basesmuma_payload_t *data_buffs;

    

};
typedef struct sm_buffer_mgmt sm_buffer_mgmt;


struct mca_bcol_basesmuma_nb_coll_buff_desc_t {
    void     *data_addr;            
    uint64_t     bank_index;        
    uint64_t     buffer_index;      
    int       active_requests;   
    ompi_request_t **requests;      
    int          data_src;           
    int          radix_mask;         
    int          radix_mask_pow;     
    int          iteration;         
    int          status;     
   	/* this is for testing */
	int 		tag;	

	volatile mca_bcol_basesmuma_ctl_struct_t **ctl_structs;
    volatile mca_bcol_basesmuma_ctl_struct_t  *my_ctl_pointer;
	volatile mca_bcol_basesmuma_ctl_struct_t  *parent_ctl_pointer; 
	volatile mca_bcol_basesmuma_ctl_struct_t  *extra_partner_ctl_pointer; 
};

typedef struct mca_bcol_basesmuma_nb_coll_buff_desc_t mca_bcol_basesmuma_nb_coll_buff_desc_t;

struct mca_bcol_basesmuma_local_mlmem_desc_t {

	uint32_t bank_index_for_release;
    struct ml_memory_block_desc_t *ml_mem_desc;
    uint32_t     num_banks;
    uint32_t     num_buffers_per_bank;
    uint32_t     size_buffer;
    uint32_t     *bank_release_counter;

	/*
	 * Number of descriptors allocated is equivalent to number of ml buffers
	 * (number of banks * number of buffers per bank) 
	 */
	mca_bcol_basesmuma_nb_coll_buff_desc_t *nb_coll_desc;
};

typedef struct mca_bcol_basesmuma_local_mlmem_desc_t mca_bcol_basesmuma_local_mlmem_desc_t;

#ifdef __PORTALS_AVAIL__
#define MAX_SM_GROUP_SIZE 32 


struct portals_scatter_allgather_nb_bcast_state_t
{
    /* local variables */
    uint64_t length;
    int my_rank, src, matched;
    int src_list[MAX_SM_GROUP_SIZE];
    int group_size;
	int64_t ready_flag;
    int pow_2, pow_2_levels;
    int src_list_index;
    uint64_t fragment_size;  /* user buffer size */

	/* Input argument variables */
	void *my_userbuf;
	int64_t sequence_number;

	/* Extra source variables */
	bool secondary_root;
	int partner , extra_partner;

	/* Scatter Allgather offsets */
	uint64_t local_sg_offset , global_sg_offset , partner_offset ;

	/* Portals messaging relevant variables */
	/* 
	 * ptl_handle_eq_t allgather_eq_h;
	 */
	ptl_handle_eq_t read_eq;
	ptl_event_t  allgather_event;
	bool msg_posted;

	/* OMPI module and component variables */
    mca_bcol_basesmuma_component_t *cs;
    struct mca_bcol_basesmuma_module_t *bcol_module; 

	/* Control structure and payload variables */
	volatile mca_bcol_basesmuma_ctl_struct_t **ctl_structs;
    volatile mca_bcol_basesmuma_ctl_struct_t  *my_ctl_pointer;
	volatile mca_bcol_basesmuma_ctl_struct_t  *parent_ctl_pointer; /* scatter source */
	volatile mca_bcol_basesmuma_ctl_struct_t  *extra_partner_ctl_pointer; /* scatter source */

	int phase;
};


typedef struct portals_scatter_allgather_nb_bcast_state_t sg_state_t;
#endif

#define SM_ARRAY_INDEX(LEAD_DIM,BUF_INDEX,PROC_INDEX) \
    ((LEAD_DIM)*(BUF_INDEX)+(PROC_INDEX))
        /* debug */
#define BARRIER_BANK_LIST_SIZE 32
        /* end debug */

struct mca_bcol_basesmuma_module_t {
    /* base structure */
    mca_bcol_base_module_t super;

    /* free list item with the control structures used for
     * the no user data collective operations
     */
    list_data_t *no_userdata_ctl;

    /* free list item with the control structures used for
     * the with user data collective operations
     */
    list_data_t *userdata_ctl;

    /*
     * information on sm control backing files for the subgroup
     * associated with this module.
     */
    bcol_basesmuma_smcm_proc_item_t **ctl_backing_files_info;

    /*
     * information on sm payload backing files for the subgroup
     * associated with this module.
     */
    bcol_basesmuma_smcm_proc_item_t **payload_backing_files_info;

    /*
     * buffers for the collective that do not involve user data -
     *   barrier, fanin, fanout.
     */
    sm_buffer_mgmt colls_no_user_data;

    /*
     * buffers for the collective with user data.
     */
    sm_buffer_mgmt colls_with_user_data;

    /* recursive-doubling tree node */
    netpatterns_pair_exchange_node_t recursive_doubling_tree;

    /* k-nomial gather/allgather tree */
    netpatterns_k_exchange_node_t knomial_allgather_tree;

    /* fanin tree node - root is rank 0 */
    netpatterns_tree_node_t fanin_node;

    /* fanout tree node - root is rank 0 */
    netpatterns_tree_node_t fanout_node;

    /* index of blocking barrier memory region to use */
    int index_blocking_barrier_memory_bank;

    /* comm to shared memory map */
    int *comm_to_sm_map;

    /* reduction fanout tree */
    netpatterns_tree_node_t* reduction_tree;

    /* broadcast fanout tree */
    netpatterns_tree_node_t* fanout_read_tree;

    /* scatter - k-ary tree */
    int scatter_kary_radix;
    netpatterns_tree_node_t *scatter_kary_tree;

	/* Knomial exchange tree */
	/* Currently used for only large message reduce */
	netpatterns_k_exchange_node_t knomial_exchange_tree;

    /* sequence number offset - want to make sure that we start
     *   id'ing collectives with id 0, so we can have simple
     *   resource management.
     */
    int64_t squence_number_offset;

    /* basesmuma specific header size into ml buffer
     * was calculated at ml level - it is the sum of 
     * all headers from all bcols and then aligned to
     * whatever alignment was requested
     */
    uint32_t total_header_size;

    /* list of possible sources */
    int *src_list;

    /* Number of possible sources */
    int src_size;

    /* smallest power of k that is smaller 
     * than or equal in size to the uma group
     */
    int pow_k_levels;

    /* size of power-of-k group */
    int pow_k;

    /* smallest power of 2 that is smaller 
     * than or equal to the smuma group size 
     */
    int pow_2_levels;

    /* size of power-of-2 group */
    int pow_2;

    /* pointer to the shared memory scratch array of each
     * process in the group.
     */
    void **shared_memory_scratch_space;

    /*
	 * Caching information for re-entrant collectives
	 */
	mca_bcol_basesmuma_local_mlmem_desc_t ml_mem;

	/*
	 * Cached offsets for lmsg reduce
	 */ 
	int **reduce_offsets;

    /*XXX:
     * Starting to explore the beauty of zero-copy for large message
     */
    struct mca_hdl_base_module_t **hdl_module;

#ifdef __PORTALS_AVAIL__
	/*
	 * Store state for NB blocking functions 
	 */
	sg_state_t sg_state;

#endif 
};

typedef struct mca_bcol_basesmuma_module_t mca_bcol_basesmuma_module_t;
OBJ_CLASS_DECLARATION(mca_bcol_basesmuma_module_t);

/* shared memory specific arguments for the bcol registration function */	
typedef struct bcol_basesmuma_registration_data_t {
    char *file_name; /* filename for payload */
    void *base_addr; /* base address to be mapped */
    size_t size;     /* size of memory block to be "registered" */
    size_t size_ctl_structure;
    size_t data_seg_alignment;
    bcol_basesmuma_smcm_mmap_t *sm_mmap; /* shared memory map struct */
    mca_coll_ml_release_buff_fn_t buff_release_cb; /* buffer release 
                                                      call back */
} bcol_basesmuma_registration_data_t;	


/* enum for signaling flag bank, when 
 * adding to this list, please keep 
 * it alphabetical
 */
/*
enum {
    ALLGATHER_FLAG,
    ALLREDUCE_FLAG,
    BARRIER_FANIN_FLAG,
    BARRIER_FANOUT_FLAG,
    BARRIER_RKING_FLAG,
    BCAST_FLAG,
    GATHER_FLAG,
    SCATTER_FLAG,
    NUM_SIGNAL_FLAGS
};

*/

enum {
    BUFFER_AVAILABLE,
    STARTED,
    FANIN,
    FANOUT
};

/* enum used for non-blocking large 
 * message bcast 
 */

enum {
   INIT,
   START,
   NOT_STARTED,
   SCATTER,
   ALLGATHER,
   EXTRA_RANK,
   PROBE,
   SCATTER_ROOT_WAIT,
   SCATTER_EXTRA_ROOT_WAIT,
   SCATTER_PARENT_WAIT,
   FINISHED
}; 

/**
 * Global component instance
 */
OMPI_MODULE_DECLSPEC extern mca_bcol_basesmuma_component_t mca_bcol_basesmuma_component;


/*
 * coll module functions
 */

/* query to see if the component is available for use, and can
 * satisfy the thread and progress requirements
 */
int mca_bcol_basesmuma_init_query(bool enable_progress_threads,
        bool enable_mpi_threads);

/* query to see if the module is available for use on the given
 * communicator, and if so, what it's priority is.
 */
mca_bcol_base_module_t **
mca_bcol_basesmuma_comm_query(mca_sbgp_base_module_t *module, int *num_modules);



/* shared memory specific memory registration function - this will be passed into the mpool */
int mca_bcol_basesmuma_register_sm(void *context_data, void *base, size_t size,
	    void **reg);	

/* shared memory specific memory deregistration function - also needed by the mpool */
int mca_bcol_basesmuma_deregister_sm(void *context_data, void *reg);

/* setup the new k_nomial tree for collectives */
int bcol_basesmuma_setup_knomial_tree(mca_bcol_base_module_t *super);

/* allocate the memory pool for the shared memory control structures */
int mca_bcol_basesmuma_allocate_pool_memory(mca_bcol_basesmuma_component_t 
		*component);

/* initialize the internal scratch buffers and control structs that will be 
   used by the module */
int base_bcol_basesmuma_setup_library_buffers(
		 		mca_bcol_basesmuma_module_t *sm_module,
		 		mca_bcol_basesmuma_component_t *cs);


/* shared memory recursive doubling initialization */
int bcol_basesmuma_rd_barrier_init(mca_bcol_base_module_t *module);

/* shared memory recusive double barrier */
int bcol_basesmuma_recursive_double_barrier(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);
/* shared memory fanin */
int bcol_basesmuma_fanin_init(mca_bcol_base_module_t *super);

/* shared memory fanout */
int bcol_basesmuma_fanout_init(mca_bcol_base_module_t *super);

/* shared memory recursive k-ing non-blocking barrier */
int bcol_basesmuma_barrier_init(mca_bcol_base_module_t *super);

/* Shared memory broadcast */
int bcol_basesmuma_bcast_init(mca_bcol_base_module_t *super);

int bcol_basesmuma_bcast(bcol_function_args_t *input_args,
    coll_ml_function_t *c_input_args);

/* Shared memory non-blocking broadcast */
int bcol_basesmuma_bcast_k_nomial_anyroot(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);

int bcol_basesmuma_bcast_k_nomial_knownroot(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);

/* Shared memory non-blocking broadcast - Large message anyroot */
int bcol_basesmuma_binary_scatter_allgather_segment(bcol_function_args_t *input_args,
            coll_ml_function_t *c_input_args);

#if 0
/*FIXME: having fun here*/
int bcol_basesmuma_hdl_zerocopy_bcast(bcol_function_args_t *input_args,
                                      coll_ml_function_t   *c_input_args);
#endif

int bcol_basesmuma_lmsg_bcast_k_nomial_anyroot(bcol_function_args_t *input_args,
    coll_ml_function_t *c_input_args);

int bcol_basesmuma_lmsg_scatter_allgather_portals_bcast(bcol_function_args_t *input_args,
    coll_ml_function_t *c_input_args);

int bcol_basesmuma_lmsg_scatter_allgather_portals_nb_bcast(bcol_function_args_t *input_args,
    coll_ml_function_t *c_input_args);

int bcol_basesmuma_lmsg_scatter_allgather_portals_nb_knownroot_bcast(bcol_function_args_t *input_args,
    coll_ml_function_t *c_input_args);

/*
 *  shared memory scatter 
 */
int bcol_basesmuma_scatter_init(mca_bcol_base_module_t *super);

/* shared memory nonblocking scatter - known root */
int bcol_basesmuma_nb_scatter_k_array_knownroot(
        bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);

/* shared memory non-blocking k-nomial barrier init */
int bcol_basesmuma_k_nomial_barrier_init(bcol_function_args_t *input_args,
                                struct coll_ml_function_t *const_args);

/* shared memory non-blocking k-nomial barrier progress */
int bcol_basesmuma_k_nomial_barrier_progress(bcol_function_args_t *input_args,
                                struct coll_ml_function_t *const_args);

/*shared memory non-blocking k-nomial allgather init */
int bcol_basesmuma_k_nomial_allgather_init(bcol_function_args_t *input_args,
                        struct coll_ml_function_t *const_args);

/* shared memory non-blocking k-nomial allgather progress */
int bcol_basesmuma_k_nomial_allgather_progress(bcol_function_args_t *input_args,
                        struct coll_ml_function_t *const_args);

/* shared memory allgather -- selection logic api */
int bcol_basesmuma_allgather_init(mca_bcol_base_module_t *super);

/* shared memory blocking k-nomial gather */
int bcol_basesmuma_k_nomial_gather(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);

/* shared memory non blocking k-nomial gather */
int bcol_basesmuma_k_nomial_gather_init(bcol_function_args_t *input_args,
                coll_ml_function_t *c_input_args);

/* shared memory non blocking k-nomial gather progress*/
int bcol_basesmuma_k_nomial_gather_progress(bcol_function_args_t *input_args,
                coll_ml_function_t *c_input_args);

/* shared memory init */
int bcol_basesmuma_gather_init(mca_bcol_base_module_t *super);
 
/* allocate shared memory control memory */
int mca_bcol_basesmuma_allocate_sm_ctl_memory(
        mca_bcol_basesmuma_component_t *cs);

/* Shared memory basesmuma reduce */
int bcol_basesmuma_reduce_init(mca_bcol_base_module_t *super);
int bcol_basesmuma_reduce_intra_fanin(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);
int bcol_basesmuma_reduce_intra_fanin_old(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);

int bcol_basesmuma_reduce_intra_reducescatter_gather(void *sbuf, void *rbuf, 
        int count, struct ompi_datatype_t *dtype, 
        struct ompi_op_t *op, 
        int root,
        struct ompi_communicator_t *comm,
        mca_coll_base_module_t *module);

/* Shared memory basesmuma allreduce */
int bcol_basesmuma_allreduce_init(mca_bcol_base_module_t *super);

int bcol_basesmuma_allreduce_intra_fanin_fanout(bcol_function_args_t *input_args,
        coll_ml_function_t *c_input_args);

int bcol_basesmuma_allreduce_intra_recursive_doubling(bcol_function_args_t *input_args, 
													  coll_ml_function_t *c_input_args);

/* initialize non-blocking barrier for recycling the memory buffers.
 *  This is not a general purpose nb_barrier, and relies on the
 *  fact that we will have only one outstanding nb-barrier per bank
 *  at a time.
 */
int bcol_basesmuma_rd_nb_barrier_init_admin(sm_nbbar_desc_t *sm_desc);

/* admin nonblocking barrier - progress function */
int bcol_basesmuma_rd_nb_barrier_progress_admin(sm_nbbar_desc_t *sm_desc);

/* Memory syncronization registration function */
int bcol_basesmuma_memsync_init(mca_bcol_base_module_t *super);

/* smcm allgather function used to exchange file offsets. */
int bcol_basesmuma_smcm_allgather_connection(
        mca_bcol_basesmuma_module_t *sm_bcol_module,
        mca_sbgp_base_module_t *module,
        opal_list_t *peer_list,
        bcol_basesmuma_smcm_proc_item_t ***backing_files,
        ompi_communicator_t *comm,
        bcol_basesmuma_smcm_file_t input, char *base_fname,
        bool map_all);

/*
 * this function initializes the internal scratch buffers and control
 * structures that will be used by the module 
 */
int base_bcol_masesmuma_setup_library_buffers(
        mca_bcol_basesmuma_module_t *sm_bcol_module, 
        mca_bcol_basesmuma_component_t *sm_bcol_component);

/* get the index of the shared memory buffer to be used */
int bcol_basesmuma_get_buff_index( sm_buffer_mgmt * buff_block,
    uint64_t buff_id );

int bcol_basesmuma_free_buff( sm_buffer_mgmt * buff_block,
    uint64_t buff_id );

/* This function does bcol_basesmuma specific memory registration and 
   issues call back for ml level bank recycling
  */ 
int bcol_basesmuma_bank_init(struct mca_coll_ml_module_t *ml_module,
		mca_bcol_base_module_t *bcol_module,
		void *reg_data);

/* bank init which is used for shared memory optimization, fall back to 
 * the bank init above if this causes problems 
 */
int bcol_basesmuma_bank_init_opti(struct mca_coll_ml_module_t *ml_module,
		mca_bcol_base_module_t *bcol_module,
		void *reg_data);

/* used for shared memory offset exchange */
int base_bcol_basesmuma_exchange_offsets(
    mca_bcol_basesmuma_module_t *sm_bcol_module,
    void **result_array, uint64_t mem_offset, int loop_limit,
    int leading_dim);


/* the progress function to be called from the opal progress function
 */
int bcol_basesmuma_progress(void);

/* Macro for initializing my shared memory control structure */
#define BASESMUMA_HEADER_INIT(my_ctl_pointer,ready_flag, sequence_number, bcol_id) \
do{                                                                                \
    int i,j;                                                                       \
    int8_t flag_offset = 0;                                                        \
    /* setup resource recycling */                                                 \
    if( my_ctl_pointer->sequence_number < sequence_number ) {                      \
        /* Signal arrival */                                                       \
        for( j = 0; j < SM_BCOLS_MAX; j++){                                        \
            my_ctl_pointer->starting_flag_value[j]=0;                              \
            for( i = 0; i < NUM_SIGNAL_FLAGS; i++){                                \
                my_ctl_pointer->flags[i][j] = -1;                                  \
            }                                                                      \
        }                                                                          \
    }                                                                              \
    /* increment the starting flag by one and return */                            \
    flag_offset = my_ctl_pointer->starting_flag_value[bcol_id];                    \
    ready_flag = flag_offset + 1;                                                  \
    MB();                                                                          \
    my_ctl_pointer->sequence_number = sequence_number;                             \
}while(0)

/* these are all the same, am using a single macro for all collectives */

#define IS_PEER_READY(peer, my_flag, my_sequence_number,flag_index, bcol_id)\
		(((peer)->sequence_number == (my_sequence_number) &&                \
		   (peer)->flags[flag_index][bcol_id] >= (my_flag))? true : false )

#if 0
#define IS_AR_DATA_READY(peer, my_flag, my_sequence_number)\
		(((peer)->sequence_number == (my_sequence_number) && \
		   (peer)->flags[ALLREDUCE_FLAG][bcol_id] >= (my_flag) \
	   	 )? true : false )

#define IS_GDATA_READY(peer, my_flag, my_sequence_number)\
		(((peer)->sequence_number == (my_sequence_number) && \
		   (peer)->flags[GATHER_FLAG][bcol_id] == (my_flag) \
	   	 )? true : false )

#define IS_PEER_READY(peer, my_flag, flag_index, my_sequence_number)\
		((((volatile int64_t)(peer)->sequence_number > (my_sequence_number)) || \
		  (((volatile int64_t)(peer)->sequence_number == (my_sequence_number)) && \
		  ((peer)->flags[flag_index][bcol_id] == (my_flag))) \
	   	 )? true : false )

#define IS_ALLREDUCE_PEER_READY(peer, my_flag, my_sequence_number)\
		 ((((volatile int64_t)(peer)->sequence_number == (my_sequence_number)) && \
		   (((peer)->flags[ALLREDUCE_FLAG][bcol_id] == (my_flag))||((peer)->flags[ALLREDUCE_FLAG][bcol_id] == (my_flag) + 1)) \
	   	 )? true : false )
#endif

#define IS_LAST_BCOL_FUNC(ml_args) \
	    ((((ml_args)->n_of_this_type_in_collective == \
		   (ml_args)->index_of_this_type_in_collective + 1 ) )? true : false)

static inline __opal_attribute_always_inline__
            size_t bcol_basesmuma_data_offset_calc(
                      mca_bcol_basesmuma_module_t *basesmuma_module)
{
    uint32_t offset = basesmuma_module->super.header_size;
    offset = ((offset + BCOL_HEAD_ALIGN - 1) / BCOL_HEAD_ALIGN) * BCOL_HEAD_ALIGN;

    return (size_t) offset;
}


END_C_DECLS

#endif /* MCA_BCOL_basesmuma_EXPORT_H */