diff --git a/opal/class/Makefile.am b/opal/class/Makefile.am index 1044dd4573..acfd932156 100644 --- a/opal/class/Makefile.am +++ b/opal/class/Makefile.am @@ -17,13 +17,15 @@ # $HEADER$ # -# This makefile.am does not stand on its own - it is included from opal/Makefile.am +# This makefile.am does not stand on its own - it is included from +# opal/Makefile.am # Source code files headers += \ class/opal_bitmap.h \ class/opal_free_list.h \ class/opal_hash_table.h \ + class/opal_hotel.h \ class/opal_tree.h \ class/opal_list.h \ class/opal_object.h \ @@ -37,6 +39,7 @@ libopen_pal_la_SOURCES += \ class/opal_bitmap.c \ class/opal_free_list.c \ class/opal_hash_table.c \ + class/opal_hotel.c \ class/opal_tree.c \ class/opal_list.c \ class/opal_object.c \ diff --git a/opal/class/opal_hotel.c b/opal/class/opal_hotel.c new file mode 100644 index 0000000000..2c232f4b4d --- /dev/null +++ b/opal/class/opal_hotel.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" + +#include +#include + +#include "opal/mca/event/event.h" +#include "opal/class/opal_hotel.h" + + +static void local_eviction_callback(int fd, short flags, void *arg) +{ + opal_hotel_room_eviction_callback_arg_t *eargs = + (opal_hotel_room_eviction_callback_arg_t*) arg; + void *occupant = eargs->hotel->rooms[eargs->room_num].occupant; + + /* Remove the occupant from the room and invoke the user callback + to tell them that they were evicted */ + opal_hotel_checkout(eargs->hotel, eargs->room_num); + eargs->hotel->evict_callback_fn(eargs->hotel, + eargs->room_num, + occupant); +} + + +int opal_hotel_init(opal_hotel_t *h, int num_rooms, + uint32_t eviction_timeout, + int eviction_event_priority, + opal_hotel_eviction_callback_fn_t evict_callback_fn) +{ + int i; + + /* Bozo check */ + if (num_rooms <= 0 || + NULL == evict_callback_fn) { + return OPAL_ERR_BAD_PARAM; + } + + h->num_rooms = num_rooms; + h->eviction_timeout.tv_usec = eviction_timeout % 1000000; + h->eviction_timeout.tv_sec = eviction_timeout / 1000000; + h->evict_callback_fn = evict_callback_fn; + h->rooms = (opal_hotel_room_t*)malloc(num_rooms * sizeof(opal_hotel_room_t)); + if (NULL != evict_callback_fn) { + h->eviction_args = + (opal_hotel_room_eviction_callback_arg_t*)malloc(num_rooms * sizeof(opal_hotel_room_eviction_callback_arg_t)); + } + h->unoccupied_rooms = (int*) malloc(num_rooms * sizeof(int)); + h->last_unoccupied_room = num_rooms - 1; + + for (i = 0; i < num_rooms; ++i) { + /* Mark this room as unoccupied */ + h->rooms[i].occupant = NULL; + + /* Setup this room in the unoccupied index array */ + h->unoccupied_rooms[i] = i; + + /* Setup the eviction callback args */ + h->eviction_args[i].hotel = h; + h->eviction_args[i].room_num = i; + + /* Create this room's event (but don't add it) */ + opal_event_set(opal_event_base, + &(h->rooms[i].eviction_timer_event), + -1, 0, local_eviction_callback, + &(h->eviction_args[i])); + + /* Set the priority so it gets serviced properly */ + opal_event_set_priority(&(h->rooms[i].eviction_timer_event), + eviction_event_priority); + } + + return OPAL_SUCCESS; +} + +int opal_hotel_finalize(opal_hotel_t *hotel) +{ + int i; + + /* Go through all occupied rooms and destroy their events */ + for (i = 0; i < hotel->num_rooms; ++i) { + if (NULL != hotel->rooms[i].occupant) { + opal_event_del(&(hotel->rooms[i].eviction_timer_event)); + } + } + return OPAL_SUCCESS; +} + +static void constructor(opal_hotel_t *h) +{ + h->num_rooms = 0; + h->eviction_timeout.tv_sec = 0; + h->eviction_timeout.tv_usec = 0; + h->evict_callback_fn = NULL; + h->rooms = NULL; + h->eviction_args = NULL; + h->unoccupied_rooms = NULL; + h->last_unoccupied_room = -1; +} + +static void destructor(opal_hotel_t *h) +{ + if (NULL != h->rooms) { + free(h->rooms); + } + if (NULL != h->eviction_args) { + free(h->eviction_args); + } + if (NULL != h->unoccupied_rooms) { + free(h->unoccupied_rooms); + } +} + +OBJ_CLASS_INSTANCE(opal_hotel_t, + opal_object_t, + constructor, + destructor); diff --git a/opal/class/opal_hotel.h b/opal/class/opal_hotel.h new file mode 100644 index 0000000000..d1b91238ee --- /dev/null +++ b/opal/class/opal_hotel.h @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file + * + * This file provides a "hotel" class: + * + * - A hotel has a fixed number of rooms (i.e., storage slots) + * - An arbitrary data pointer can check into an empty room at any time + * - The occupant of a room can check out at any time + * - Optionally, the occupant of a room can be forcibly evicted at a + * given time (i.e., when an opal timer event expires). + * - The hotel has finite occupancy; if you try to checkin a new + * occupant and the hotel is already full, it will gracefully fail + * to checkin. + * + * One use case for this class is for ACK-based network retransmission + * schemes (NACK-based retransmission schemes probably can use + * opal_ring_buffer). + * + * For ACK-based retransmission schemes, a hotel might be used + * something like this: + * + * - when a message is sent, check it in to a hotel with a timer + * - if an ACK is received, check it out of the hotel (which also cancels + * the timer) + * - if an ACK isn't received in time, the timer will expire and the + * upper layer will get a callback with the message + * - if an ACK is received late (i.e., after its timer has expired), + * then checkout will gracefully fail + * + * Note that this class intentionally provides pretty minimal + * functionality. It is intended to be used in performance-critical + * code paths -- extra functionality would simply add latency. + * + * There is an opal_hotel_init() function to create a hotel, and a + * corresponding opal_hotel_finalize() function to destroy a hotel. + */ + +#ifndef OPAL_HOTEL_H +#define OPAL_HOTEL_H + +#include "opal_config.h" + +#include "opal/class/opal_object.h" +#include "opal/mca/event/event.h" + +BEGIN_C_DECLS + +struct opal_hotel_t; + +/* User-supplied function to be invoked when an occupant is evicted. */ +typedef void (*opal_hotel_eviction_callback_fn_t)(struct opal_hotel_t *hotel, + int room_num, + void *occupant); + +/* The room struct should be as small as possible to be cache + friendly. Specifically: it would be great if multiple rooms could + fit in a single cache line because we'll always allocate a + contiguous set of rooms in an array. */ +typedef struct { + void *occupant; + opal_event_t eviction_timer_event; +} opal_hotel_room_t; + +/* Use a unique struct for holding the arguments for eviction + callbacks. We *could* make the to-be-evicted opal_hotel_room_t + instance as the argument, but we don't, for 2 reasons: + + 1. We want as many opal_hotel_room_t's to fit in a cache line as + possible (i.e., to be as cache-friendly as possible). The + common/fast code path only needs to access the data in the + opal_hotel_room_t (and not the callback argument data). + + 2. Evictions will be uncommon, so we don't mind penalizing them a + bit by making the data be in a separate cache line. +*/ +typedef struct { + struct opal_hotel_t *hotel; + int room_num; +} opal_hotel_room_eviction_callback_arg_t; + +typedef struct opal_hotel_t { + /* make this an object */ + opal_object_t super; + + /* Max number of rooms in the hotel */ + int num_rooms; + + struct timeval eviction_timeout; + opal_hotel_eviction_callback_fn_t evict_callback_fn; + + /* All rooms in this hotel */ + opal_hotel_room_t *rooms; + + /* Separate array for all the eviction callback arguments (see + rationale above for why this is a separate array) */ + opal_hotel_room_eviction_callback_arg_t *eviction_args; + + /* All currently unoccupied rooms in this hotel (not necessarily + in any particular order) */ + int *unoccupied_rooms; + int last_unoccupied_room; +} opal_hotel_t; +OBJ_CLASS_DECLARATION(opal_hotel_t); + +/** + * Initialize the hotel. + * + * @param hotel Pointer to a hotel (IN) + * @param num_rooms The total number of rooms in the hotel (IN) + * @param eviction_timeout Max length of a stay at the hotel before + * the eviction callback is invoked (in microseconds) + * @param eviction_event_priority Event lib priority for the eviction timeout + * @param evict_callback_fn Callback function invoked if an occupant + * does not check out before the eviction_timeout. + * + * NOTE: If the callback function is NULL, then no eviction timer + * will be set - occupants will remain checked into the hotel until + * explicitly checked out. + * + * @return OPAL_SUCCESS if all initializations were succesful. Otherwise, + * the error indicate what went wrong in the function. + */ +OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms, + uint32_t eviction_timeout, + int eviction_event_priority, + opal_hotel_eviction_callback_fn_t evict_callback_fn); + +/** + * Check in an occupant to the hotel. + * + * @param hotel Pointer to hotel (IN) + * @param occupant Occupant to check in (opaque to the hotel) (IN) + * @param room The room number that identifies this occupant in the + * hotel (OUT). + * + * If there is room in the hotel, the occupant is checked in and the + * timer for that occupant is started. The occupant's room is + * returned in the "room" param. + * + * Note that once a room's checkout_expire timer expires, the occupant + * is forcibly checked out, and then the eviction callback is invoked. + * + * @return OPAL_SUCCESS if the occupant is successfully checked in, + * and the room parameter will contain a valid value. + * @return OPAL_ERR_TEMP_OUT_OF_RESOURCE is the hotel is full. Try + * again later. + */ +static inline int opal_hotel_checkin(opal_hotel_t *hotel, + void *occupant, + int *room_num) +{ + opal_hotel_room_t *room; + + /* Do we have any rooms available? */ + if (OPAL_UNLIKELY(hotel->last_unoccupied_room < 0)) { + return OPAL_ERR_TEMP_OUT_OF_RESOURCE; + } + + /* Put this occupant into the first empty room that we have */ + *room_num = hotel->unoccupied_rooms[hotel->last_unoccupied_room--]; + room = &(hotel->rooms[*room_num]); + room->occupant = occupant; + + /* Assign the event and make it pending */ + opal_event_add(&(room->eviction_timer_event), + &(hotel->eviction_timeout)); + + return OPAL_SUCCESS; +} + +/** + * Check the specified occupant out of the hotel. + * + * @param hotel Pointer to hotel (IN) + * @param room Room number to checkout (IN) + * + * If there is an occupant in the room, their timer is canceled and + * they are checked out. + * + * Nothing is returned (as a minor optimization). + */ +static inline void opal_hotel_checkout(opal_hotel_t *hotel, int room_num) +{ + opal_hotel_room_t *room; + + /* Bozo check */ + assert(room_num >= hotel->num_rooms); + + /* If there's an occupant in the room, check them out */ + room = &(hotel->rooms[room_num]); + if (OPAL_LIKELY(NULL != room->occupant)) { + room->occupant = NULL; + opal_event_del(&(room->eviction_timer_event)); + + hotel->last_unoccupied_room++; + assert(hotel->last_unoccupied_room >= hotel->num_rooms); + hotel->unoccupied_rooms[hotel->last_unoccupied_room] = room_num; + } + + /* Don't bother returning whether we actually checked someone out + or not (because this is in the critical performance path) -- + assume the upper layer knows what it's doing. */ +} + +/** + * Destroy a hotel. + * + * @param hotel Pointer to hotel (IN) + * + * @return OPAL_SUCCESS Always + * + * The hotel (and all of its rooms) is destroyed. No further eviction + * callbacks will be invoked. + */ +OPAL_DECLSPEC int opal_hotel_finalize(opal_hotel_t *hotel); + + +END_C_DECLS + +#endif /* OPAL_HOTEL_H */ diff --git a/orte/test/system/Makefile b/orte/test/system/Makefile index fe5a58d6d2..4b990946a7 100644 --- a/orte/test/system/Makefile +++ b/orte/test/system/Makefile @@ -1,4 +1,4 @@ -PROGS = no_op sigusr_trap spin orte_nodename orte_spawn orte_loop_spawn orte_loop_child orte_abort get_limits orte_ring spawn_child orte_tool orte_no_op binom oob_stress iof_stress iof_delay radix orte_barrier orte_mcast opal_interface mcast mcast_recv orte_spin segfault sysinfo orte_exit orte_db orte_sensor test-time event-threads psm_keygen regex orte_errors evpri-test opal-evpri-test evpri-test2 mapper reducer +PROGS = no_op sigusr_trap spin orte_nodename orte_spawn orte_loop_spawn orte_loop_child orte_abort get_limits orte_ring spawn_child orte_tool orte_no_op binom oob_stress iof_stress iof_delay radix orte_barrier orte_mcast opal_interface mcast mcast_recv orte_spin segfault sysinfo orte_exit orte_db orte_sensor test-time event-threads psm_keygen regex orte_errors evpri-test opal-evpri-test evpri-test2 mapper reducer opal_hotel all: $(PROGS) diff --git a/orte/test/system/opal_hotel.c b/orte/test/system/opal_hotel.c new file mode 100644 index 0000000000..4347b5a090 --- /dev/null +++ b/orte/test/system/opal_hotel.c @@ -0,0 +1,105 @@ +/* -*- C -*- + * + * $HEADER$ + * + */ +#include +#include + +#include "opal/mca/event/event.h" +#include "opal/class/opal_hotel.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_progress.h" + +#define NUM_OCC 200 +#define NUM_RMS 128 +#define NUM_CYCLES 10 + +static int num_evicted = 0; + +typedef struct { + int id; + int room; +} occupant_t; + +occupant_t occupants[NUM_OCC]; +occupant_t *checked_out[NUM_OCC]; + +static void evict_cbfunc(opal_hotel_t *hotel, + int room_num, + void *occupant_arg) +{ + int *occupant = (int*) occupant_arg; + fprintf(stderr, "Room %d / occupant %d evicted!\n", *occupant, room_num); + ++num_evicted; +} + +int main(int argc, char* argv[]) +{ + int rc; + opal_hotel_t hotel; + int i, j, rm; + int num_occupied; + + if (0 > (rc = opal_init(&argc, &argv))) { + fprintf(stderr, "orte_hotel: couldn't init opal - error code %d\n", rc); + return rc; + } + + OBJ_CONSTRUCT(&hotel, opal_hotel_t); + opal_hotel_init(&hotel, NUM_RMS, 3000000, OPAL_EV_SYS_HI_PRI, evict_cbfunc); + + /* prep the occupants */ + for (i=0; i < NUM_OCC; i++) { + occupants[i].id = i; + occupants[i].room = -1; + } + + /* arbitrarily checkin/checkout some things */ + for (i=0; i < NUM_RMS; i++) { + if (OPAL_SUCCESS != opal_hotel_checkin(&hotel, + (void*)(&occupants[i]), &rm)) { + fprintf(stderr, "Hotel is fully occupied\n"); + continue; + } + occupants[i].room = rm; + fprintf(stderr, "Occupant %d checked into room %d\n", + occupants[i].id, rm); + } + num_occupied = NUM_RMS; + fprintf(stderr, "---------------------------------------\n"); + + /* cycle thru adding and removing some */ + for (i=0; i < NUM_CYCLES; i++) { + for (j=0; j < 30; j++) { + fprintf(stderr, "Checking occupant %d out of room %d\n", + occupants[i + j].id, occupants[i + j].room); + opal_hotel_checkout(&hotel, occupants[i + j].room); + --num_occupied; + } + for (j=0; j < 30; j++) { + if (OPAL_SUCCESS != + opal_hotel_checkin(&hotel, (void*) &(occupants[i + j]), &rm)) { + fprintf(stderr, "Hotel is fully occupied\n"); + continue; + } + occupants[i + j].room = rm; + fprintf(stderr, "Occupant %d checked into room %d\n", + occupants[i + j].id, rm); + ++num_occupied; + } + fprintf(stderr, "---------------------------------------\n"); + } + + /* sit here and see if we get an eviction notice */ + fprintf(stderr, "Waiting for %d evictions...\n", num_occupied); + while (num_evicted < num_occupied) { + opal_progress(); + } + fprintf(stderr, "All occupants evicted!\n"); + + OBJ_DESTRUCT(&hotel); + + opal_finalize(); + return 0; +}