openmpi/opal/class/opal_hotel.h

/*
 * Copyright (c) 2012      Cisco Systems, Inc. All rights reserved.
 * Copyright (c) 2012      Los Alamos National Security, LLC. All rights reserved
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */

/** @file
 *
 * This file provides a "hotel" class:
 *
 * - A hotel has a fixed number of rooms (i.e., storage slots)
 * - An arbitrary data pointer can check into an empty room at any time
 * - The occupant of a room can check out at any time
 * - Optionally, the occupant of a room can be forcibly evicted at a
 *   given time (i.e., when an opal timer event expires).
 * - The hotel has finite occupancy; if you try to checkin a new
 *   occupant and the hotel is already full, it will gracefully fail
 *   to checkin.
 *
 * One use case for this class is for ACK-based network retransmission
 * schemes (NACK-based retransmission schemes probably can use
 * opal_ring_buffer).  
 *
 * For ACK-based retransmission schemes, a hotel might be used
 * something like this:
 *
 * - when a message is sent, check it in to a hotel with a timer
 * - if an ACK is received, check it out of the hotel (which also cancels
 *   the timer)
 * - if an ACK isn't received in time, the timer will expire and the
 *   upper layer will get a callback with the message
 * - if an ACK is received late (i.e., after its timer has expired),
 *   then checkout will gracefully fail
 *
 * Note that this class intentionally provides pretty minimal
 * functionality.  It is intended to be used in performance-critical
 * code paths -- extra functionality would simply add latency.
 *
 * There is an opal_hotel_init() function to create a hotel, and a
 * corresponding opal_hotel_finalize() function to destroy a hotel.
 */

#ifndef OPAL_HOTEL_H
#define OPAL_HOTEL_H

#include "opal_config.h"

#include "opal/class/opal_object.h"
#include "opal/mca/event/event.h"

BEGIN_C_DECLS

struct opal_hotel_t;

/* User-supplied function to be invoked when an occupant is evicted. */
typedef void (*opal_hotel_eviction_callback_fn_t)(struct opal_hotel_t *hotel, 
                                                  int room_num,
                                                  void *occupant);

/* The room struct should be as small as possible to be cache
   friendly.  Specifically: it would be great if multiple rooms could
   fit in a single cache line because we'll always allocate a
   contiguous set of rooms in an array. */
typedef struct {
    void *occupant;
    opal_event_t eviction_timer_event;
} opal_hotel_room_t;

/* Use a unique struct for holding the arguments for eviction
   callbacks.  We *could* make the to-be-evicted opal_hotel_room_t
   instance as the argument, but we don't, for 2 reasons:

   1. We want as many opal_hotel_room_t's to fit in a cache line as
      possible (i.e., to be as cache-friendly as possible).  The
      common/fast code path only needs to access the data in the
      opal_hotel_room_t (and not the callback argument data).

   2. Evictions will be uncommon, so we don't mind penalizing them a
      bit by making the data be in a separate cache line.
*/
typedef struct {
    struct opal_hotel_t *hotel;
    int room_num;
} opal_hotel_room_eviction_callback_arg_t;

typedef struct opal_hotel_t {
    /* make this an object */
    opal_object_t super;

    /* Max number of rooms in the hotel */
    int num_rooms;

    struct timeval eviction_timeout;
    opal_hotel_eviction_callback_fn_t evict_callback_fn;

    /* All rooms in this hotel */
    opal_hotel_room_t *rooms;

    /* Separate array for all the eviction callback arguments (see
       rationale above for why this is a separate array) */
    opal_hotel_room_eviction_callback_arg_t *eviction_args;

    /* All currently unoccupied rooms in this hotel (not necessarily
       in any particular order) */
    int *unoccupied_rooms;
    int last_unoccupied_room;
} opal_hotel_t;
OBJ_CLASS_DECLARATION(opal_hotel_t);

/**
 * Initialize the hotel.
 *
 * @param hotel Pointer to a hotel (IN)
 * @param num_rooms The total number of rooms in the hotel (IN)
 * @param eviction_timeout Max length of a stay at the hotel before
 * the eviction callback is invoked (in microseconds)
 * @param eviction_event_priority Event lib priority for the eviction timeout
 * @param evict_callback_fn Callback function invoked if an occupant
 * does not check out before the eviction_timeout.
 *
 * NOTE: If the callback function is NULL, then no eviction timer
 * will be set - occupants will remain checked into the hotel until
 * explicitly checked out.
 *
 * @return OPAL_SUCCESS if all initializations were succesful. Otherwise,
 *  the error indicate what went wrong in the function.
 */
OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms,
                                  uint32_t eviction_timeout,
                                  int eviction_event_priority,
                                  opal_hotel_eviction_callback_fn_t evict_callback_fn);

/**
 * Check in an occupant to the hotel.
 *
 * @param hotel Pointer to hotel (IN)
 * @param occupant Occupant to check in (opaque to the hotel) (IN)
 * @param room The room number that identifies this occupant in the
 * hotel (OUT).
 *
 * If there is room in the hotel, the occupant is checked in and the
 * timer for that occupant is started.  The occupant's room is
 * returned in the "room" param.
 *
 * Note that once a room's checkout_expire timer expires, the occupant
 * is forcibly checked out, and then the eviction callback is invoked.
 *
 * @return OPAL_SUCCESS if the occupant is successfully checked in,
 * and the room parameter will contain a valid value.
 * @return OPAL_ERR_TEMP_OUT_OF_RESOURCE is the hotel is full.  Try
 * again later.
 */
static inline int opal_hotel_checkin(opal_hotel_t *hotel,
                                     void *occupant,
                                     int *room_num)
{
    opal_hotel_room_t *room;

    /* Do we have any rooms available? */
    if (OPAL_UNLIKELY(hotel->last_unoccupied_room < 0)) {
        return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
    }

    /* Put this occupant into the first empty room that we have */
    *room_num = hotel->unoccupied_rooms[hotel->last_unoccupied_room--];
    room = &(hotel->rooms[*room_num]);
    room->occupant = occupant;

    /* Assign the event and make it pending */
    opal_event_add(&(room->eviction_timer_event),
                   &(hotel->eviction_timeout));

    return OPAL_SUCCESS;
}

/**
 * Check the specified occupant out of the hotel.
 *
 * @param hotel Pointer to hotel (IN)
 * @param room Room number to checkout (IN)
 *
 * If there is an occupant in the room, their timer is canceled and
 * they are checked out.
 *
 * Nothing is returned (as a minor optimization).
 */
static inline void opal_hotel_checkout(opal_hotel_t *hotel, int room_num)
{
    opal_hotel_room_t *room;

    /* Bozo check */
    assert(room_num >= hotel->num_rooms);

    /* If there's an occupant in the room, check them out */
    room = &(hotel->rooms[room_num]);
    if (OPAL_LIKELY(NULL != room->occupant)) {
        room->occupant = NULL;
        opal_event_del(&(room->eviction_timer_event));

        hotel->last_unoccupied_room++;
        assert(hotel->last_unoccupied_room >= hotel->num_rooms);
        hotel->unoccupied_rooms[hotel->last_unoccupied_room] = room_num;
    }

    /* Don't bother returning whether we actually checked someone out
       or not (because this is in the critical performance path) --
       assume the upper layer knows what it's doing. */
}

/**
 * Destroy a hotel.
 *
 * @param hotel Pointer to hotel (IN)
 *
 * @return OPAL_SUCCESS Always
 *
 * The hotel (and all of its rooms) is destroyed.  No further eviction
 * callbacks will be invoked.
 */
OPAL_DECLSPEC int opal_hotel_finalize(opal_hotel_t *hotel);


END_C_DECLS

#endif /* OPAL_HOTEL_H */