Add new "opal_hotel" class. Abstractly speaking, this class does the

following:

 * Provides a fixed number of resource slots (i.e., "hotel rooms").
 * Allows one thing to occupy a resource slot at a time (i.e., each
   hotel room can have an occupant check in to that room).
 * Resource slots can be vacated at any time (i.e., occupants can
   voluntarily check out of their hotel room).
 * Resource slots can be occupied for a specific maximum amount of
   time.  If that time expires, the occupant is forcibly evicted and
   the upper layer is notified via (libevent) callback (i.e., the maid
   will kick an occupant of out of their room when their reservation
   is over).

This class can be to be used for things like retransmission schemes
for unreliable transports.  For example, a message sent on an
unreliable transport can be checked in to a hotel room.  If an ACK for
that message is received, the message can be checked out.  But if the
ACK is never received, the message will eventually be evicted from its
room and the upper layer will be notified that the message failed to
check out in time (i.e., that an ACK for that message was not received
in time).

Code using this class is currently being developed off-trunk, but will
be coming to SVN soon.

This commit was SVN r27067.

opal/class/Makefile.am

@@ -17,13 +17,15 @@
 # $HEADER$
 #
 
-# This makefile.am does not stand on its own - it is included from opal/Makefile.am
+# This makefile.am does not stand on its own - it is included from
+# opal/Makefile.am
 
 # Source code files
 headers += \
         class/opal_bitmap.h \
         class/opal_free_list.h \
         class/opal_hash_table.h \
+        class/opal_hotel.h \
         class/opal_tree.h \
         class/opal_list.h \
         class/opal_object.h \
@@ -37,6 +39,7 @@ libopen_pal_la_SOURCES += \
         class/opal_bitmap.c \
         class/opal_free_list.c \
         class/opal_hash_table.c \
+        class/opal_hotel.c \
         class/opal_tree.c \
         class/opal_list.c \
         class/opal_object.c \

opal/class/opal_hotel.c

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2012      Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC. All rights reserved
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <stdio.h>
+#include <stddef.h>
+
+#include "opal/mca/event/event.h"
+#include "opal/class/opal_hotel.h"
+
+
+static void local_eviction_callback(int fd, short flags, void *arg)
+{
+    opal_hotel_room_eviction_callback_arg_t *eargs = 
+        (opal_hotel_room_eviction_callback_arg_t*) arg;
+    void *occupant = eargs->hotel->rooms[eargs->room_num].occupant;
+
+    /* Remove the occupant from the room and invoke the user callback
+       to tell them that they were evicted */
+    opal_hotel_checkout(eargs->hotel, eargs->room_num);
+    eargs->hotel->evict_callback_fn(eargs->hotel,
+                                    eargs->room_num,
+                                    occupant);
+}
+
+
+int opal_hotel_init(opal_hotel_t *h, int num_rooms,
+                    uint32_t eviction_timeout,
+                    int eviction_event_priority,
+                    opal_hotel_eviction_callback_fn_t evict_callback_fn)
+{
+    int i;
+
+    /* Bozo check */
+    if (num_rooms <= 0 ||
+        NULL == evict_callback_fn) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    h->num_rooms = num_rooms;
+    h->eviction_timeout.tv_usec = eviction_timeout % 1000000;
+    h->eviction_timeout.tv_sec = eviction_timeout / 1000000;
+    h->evict_callback_fn = evict_callback_fn;
+    h->rooms = (opal_hotel_room_t*)malloc(num_rooms * sizeof(opal_hotel_room_t));
+    if (NULL != evict_callback_fn) {
+        h->eviction_args = 
+            (opal_hotel_room_eviction_callback_arg_t*)malloc(num_rooms * sizeof(opal_hotel_room_eviction_callback_arg_t));
+    }
+    h->unoccupied_rooms = (int*) malloc(num_rooms * sizeof(int));
+    h->last_unoccupied_room = num_rooms - 1;
+
+    for (i = 0; i < num_rooms; ++i) {
+        /* Mark this room as unoccupied */
+        h->rooms[i].occupant = NULL;
+
+        /* Setup this room in the unoccupied index array */
+        h->unoccupied_rooms[i] = i;
+
+        /* Setup the eviction callback args */
+        h->eviction_args[i].hotel = h;
+        h->eviction_args[i].room_num = i;
+
+        /* Create this room's event (but don't add it) */
+        opal_event_set(opal_event_base,
+                       &(h->rooms[i].eviction_timer_event),
+                       -1, 0, local_eviction_callback,
+                       &(h->eviction_args[i]));
+
+        /* Set the priority so it gets serviced properly */
+        opal_event_set_priority(&(h->rooms[i].eviction_timer_event),
+                                eviction_event_priority);
+    }
+
+    return OPAL_SUCCESS;
+}
+
+int opal_hotel_finalize(opal_hotel_t *hotel)
+{
+    int i;
+
+    /* Go through all occupied rooms and destroy their events */
+    for (i = 0; i < hotel->num_rooms; ++i) {
+        if (NULL != hotel->rooms[i].occupant) {
+            opal_event_del(&(hotel->rooms[i].eviction_timer_event));
+        }
+    }
+    return OPAL_SUCCESS;
+}
+
+static void constructor(opal_hotel_t *h)
+{
+    h->num_rooms = 0;
+    h->eviction_timeout.tv_sec = 0;
+    h->eviction_timeout.tv_usec = 0;
+    h->evict_callback_fn = NULL;
+    h->rooms = NULL;
+    h->eviction_args = NULL;
+    h->unoccupied_rooms = NULL;
+    h->last_unoccupied_room = -1;
+}
+
+static void destructor(opal_hotel_t *h)
+{
+    if (NULL != h->rooms) {
+        free(h->rooms);
+    }
+    if (NULL != h->eviction_args) {
+        free(h->eviction_args);
+    }
+    if (NULL != h->unoccupied_rooms) {
+        free(h->unoccupied_rooms);
+    }
+}
+
+OBJ_CLASS_INSTANCE(opal_hotel_t,
+                   opal_object_t,
+                   constructor,
+                   destructor);

opal/class/opal_hotel.h

@@ -0,0 +1,229 @@
+/*
+ * Copyright (c) 2012      Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC. All rights reserved
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+/** @file
+ *
+ * This file provides a "hotel" class:
+ *
+ * - A hotel has a fixed number of rooms (i.e., storage slots)
+ * - An arbitrary data pointer can check into an empty room at any time
+ * - The occupant of a room can check out at any time
+ * - Optionally, the occupant of a room can be forcibly evicted at a
+ *   given time (i.e., when an opal timer event expires).
+ * - The hotel has finite occupancy; if you try to checkin a new
+ *   occupant and the hotel is already full, it will gracefully fail
+ *   to checkin.
+ *
+ * One use case for this class is for ACK-based network retransmission
+ * schemes (NACK-based retransmission schemes probably can use
+ * opal_ring_buffer).  
+ *
+ * For ACK-based retransmission schemes, a hotel might be used
+ * something like this:
+ *
+ * - when a message is sent, check it in to a hotel with a timer
+ * - if an ACK is received, check it out of the hotel (which also cancels
+ *   the timer)
+ * - if an ACK isn't received in time, the timer will expire and the
+ *   upper layer will get a callback with the message
+ * - if an ACK is received late (i.e., after its timer has expired),
+ *   then checkout will gracefully fail
+ *
+ * Note that this class intentionally provides pretty minimal
+ * functionality.  It is intended to be used in performance-critical
+ * code paths -- extra functionality would simply add latency.
+ *
+ * There is an opal_hotel_init() function to create a hotel, and a
+ * corresponding opal_hotel_finalize() function to destroy a hotel.
+ */
+
+#ifndef OPAL_HOTEL_H
+#define OPAL_HOTEL_H
+
+#include "opal_config.h"
+
+#include "opal/class/opal_object.h"
+#include "opal/mca/event/event.h"
+
+BEGIN_C_DECLS
+
+struct opal_hotel_t;
+
+/* User-supplied function to be invoked when an occupant is evicted. */
+typedef void (*opal_hotel_eviction_callback_fn_t)(struct opal_hotel_t *hotel, 
+                                                  int room_num,
+                                                  void *occupant);
+
+/* The room struct should be as small as possible to be cache
+   friendly.  Specifically: it would be great if multiple rooms could
+   fit in a single cache line because we'll always allocate a
+   contiguous set of rooms in an array. */
+typedef struct {
+    void *occupant;
+    opal_event_t eviction_timer_event;
+} opal_hotel_room_t;
+
+/* Use a unique struct for holding the arguments for eviction
+   callbacks.  We *could* make the to-be-evicted opal_hotel_room_t
+   instance as the argument, but we don't, for 2 reasons:
+
+   1. We want as many opal_hotel_room_t's to fit in a cache line as
+      possible (i.e., to be as cache-friendly as possible).  The
+      common/fast code path only needs to access the data in the
+      opal_hotel_room_t (and not the callback argument data).
+
+   2. Evictions will be uncommon, so we don't mind penalizing them a
+      bit by making the data be in a separate cache line.
+*/
+typedef struct {
+    struct opal_hotel_t *hotel;
+    int room_num;
+} opal_hotel_room_eviction_callback_arg_t;
+
+typedef struct opal_hotel_t {
+    /* make this an object */
+    opal_object_t super;
+
+    /* Max number of rooms in the hotel */
+    int num_rooms;
+
+    struct timeval eviction_timeout;
+    opal_hotel_eviction_callback_fn_t evict_callback_fn;
+
+    /* All rooms in this hotel */
+    opal_hotel_room_t *rooms;
+
+    /* Separate array for all the eviction callback arguments (see
+       rationale above for why this is a separate array) */
+    opal_hotel_room_eviction_callback_arg_t *eviction_args;
+
+    /* All currently unoccupied rooms in this hotel (not necessarily
+       in any particular order) */
+    int *unoccupied_rooms;
+    int last_unoccupied_room;
+} opal_hotel_t;
+OBJ_CLASS_DECLARATION(opal_hotel_t);
+
+/**
+ * Initialize the hotel.
+ *
+ * @param hotel Pointer to a hotel (IN)
+ * @param num_rooms The total number of rooms in the hotel (IN)
+ * @param eviction_timeout Max length of a stay at the hotel before
+ * the eviction callback is invoked (in microseconds)
+ * @param eviction_event_priority Event lib priority for the eviction timeout
+ * @param evict_callback_fn Callback function invoked if an occupant
+ * does not check out before the eviction_timeout.
+ *
+ * NOTE: If the callback function is NULL, then no eviction timer
+ * will be set - occupants will remain checked into the hotel until
+ * explicitly checked out.
+ *
+ * @return OPAL_SUCCESS if all initializations were succesful. Otherwise,
+ *  the error indicate what went wrong in the function.
+ */
+OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms,
+                                  uint32_t eviction_timeout,
+                                  int eviction_event_priority,
+                                  opal_hotel_eviction_callback_fn_t evict_callback_fn);
+
+/**
+ * Check in an occupant to the hotel.
+ *
+ * @param hotel Pointer to hotel (IN)
+ * @param occupant Occupant to check in (opaque to the hotel) (IN)
+ * @param room The room number that identifies this occupant in the
+ * hotel (OUT).
+ *
+ * If there is room in the hotel, the occupant is checked in and the
+ * timer for that occupant is started.  The occupant's room is
+ * returned in the "room" param.
+ *
+ * Note that once a room's checkout_expire timer expires, the occupant
+ * is forcibly checked out, and then the eviction callback is invoked.
+ *
+ * @return OPAL_SUCCESS if the occupant is successfully checked in,
+ * and the room parameter will contain a valid value.
+ * @return OPAL_ERR_TEMP_OUT_OF_RESOURCE is the hotel is full.  Try
+ * again later.
+ */
+static inline int opal_hotel_checkin(opal_hotel_t *hotel,
+                                     void *occupant,
+                                     int *room_num)
+{
+    opal_hotel_room_t *room;
+
+    /* Do we have any rooms available? */
+    if (OPAL_UNLIKELY(hotel->last_unoccupied_room < 0)) {
+        return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
+    }
+
+    /* Put this occupant into the first empty room that we have */
+    *room_num = hotel->unoccupied_rooms[hotel->last_unoccupied_room--];
+    room = &(hotel->rooms[*room_num]);
+    room->occupant = occupant;
+
+    /* Assign the event and make it pending */
+    opal_event_add(&(room->eviction_timer_event),
+                   &(hotel->eviction_timeout));
+
+    return OPAL_SUCCESS;
+}
+
+/**
+ * Check the specified occupant out of the hotel.
+ *
+ * @param hotel Pointer to hotel (IN)
+ * @param room Room number to checkout (IN)
+ *
+ * If there is an occupant in the room, their timer is canceled and
+ * they are checked out.
+ *
+ * Nothing is returned (as a minor optimization).
+ */
+static inline void opal_hotel_checkout(opal_hotel_t *hotel, int room_num)
+{
+    opal_hotel_room_t *room;
+
+    /* Bozo check */
+    assert(room_num >= hotel->num_rooms);
+
+    /* If there's an occupant in the room, check them out */
+    room = &(hotel->rooms[room_num]);
+    if (OPAL_LIKELY(NULL != room->occupant)) {
+        room->occupant = NULL;
+        opal_event_del(&(room->eviction_timer_event));
+
+        hotel->last_unoccupied_room++;
+        assert(hotel->last_unoccupied_room >= hotel->num_rooms);
+        hotel->unoccupied_rooms[hotel->last_unoccupied_room] = room_num;
+    }
+
+    /* Don't bother returning whether we actually checked someone out
+       or not (because this is in the critical performance path) --
+       assume the upper layer knows what it's doing. */
+}
+
+/**
+ * Destroy a hotel.
+ *
+ * @param hotel Pointer to hotel (IN)
+ *
+ * @return OPAL_SUCCESS Always
+ *
+ * The hotel (and all of its rooms) is destroyed.  No further eviction
+ * callbacks will be invoked.
+ */
+OPAL_DECLSPEC int opal_hotel_finalize(opal_hotel_t *hotel);
+
+
+END_C_DECLS
+
+#endif /* OPAL_HOTEL_H */

orte/test/system/Makefile

@@ -1,4 +1,4 @@
-PROGS = no_op sigusr_trap spin orte_nodename orte_spawn orte_loop_spawn orte_loop_child orte_abort get_limits orte_ring spawn_child orte_tool orte_no_op binom oob_stress iof_stress iof_delay radix orte_barrier orte_mcast opal_interface mcast mcast_recv orte_spin segfault sysinfo orte_exit orte_db orte_sensor test-time event-threads psm_keygen regex orte_errors evpri-test opal-evpri-test evpri-test2 mapper reducer
+PROGS = no_op sigusr_trap spin orte_nodename orte_spawn orte_loop_spawn orte_loop_child orte_abort get_limits orte_ring spawn_child orte_tool orte_no_op binom oob_stress iof_stress iof_delay radix orte_barrier orte_mcast opal_interface mcast mcast_recv orte_spin segfault sysinfo orte_exit orte_db orte_sensor test-time event-threads psm_keygen regex orte_errors evpri-test opal-evpri-test evpri-test2 mapper reducer opal_hotel
 
 all: $(PROGS)
 

orte/test/system/opal_hotel.c

@@ -0,0 +1,105 @@
+/* -*- C -*-
+ *
+ * $HEADER$
+ *
+ */
+#include <stdio.h>
+#include <unistd.h>
+
+#include "opal/mca/event/event.h"
+#include "opal/class/opal_hotel.h"
+#include "opal/runtime/opal.h"
+#include "opal/runtime/opal_progress.h"
+
+#define NUM_OCC 200
+#define NUM_RMS 128
+#define NUM_CYCLES 10
+
+static int num_evicted = 0;
+
+typedef struct {
+    int id;
+    int room;
+} occupant_t;
+
+occupant_t occupants[NUM_OCC];
+occupant_t *checked_out[NUM_OCC];
+
+static void evict_cbfunc(opal_hotel_t *hotel, 
+                         int room_num,
+                         void *occupant_arg)
+{
+    int *occupant = (int*) occupant_arg;
+    fprintf(stderr, "Room %d / occupant %d evicted!\n", *occupant, room_num);
+    ++num_evicted;
+}
+
+int main(int argc, char* argv[])
+{
+    int rc;
+    opal_hotel_t hotel;
+    int i, j, rm;
+    int num_occupied;
+
+    if (0 > (rc = opal_init(&argc, &argv))) {
+        fprintf(stderr, "orte_hotel: couldn't init opal - error code %d\n", rc);
+        return rc;
+    }
+    
+    OBJ_CONSTRUCT(&hotel, opal_hotel_t);
+    opal_hotel_init(&hotel, NUM_RMS, 3000000, OPAL_EV_SYS_HI_PRI, evict_cbfunc);
+
+    /* prep the occupants */
+    for (i=0; i < NUM_OCC; i++) {
+        occupants[i].id = i;
+        occupants[i].room = -1;
+    }
+
+    /* arbitrarily checkin/checkout some things */
+    for (i=0; i < NUM_RMS; i++) {
+        if (OPAL_SUCCESS != opal_hotel_checkin(&hotel, 
+                                               (void*)(&occupants[i]), &rm)) {
+            fprintf(stderr, "Hotel is fully occupied\n");
+            continue;
+        }
+        occupants[i].room = rm;
+        fprintf(stderr, "Occupant %d checked into room %d\n", 
+                occupants[i].id, rm);
+    }
+    num_occupied = NUM_RMS;
+    fprintf(stderr, "---------------------------------------\n");
+
+    /* cycle thru adding and removing some */
+    for (i=0; i < NUM_CYCLES; i++) {
+        for (j=0; j < 30; j++) {
+            fprintf(stderr, "Checking occupant %d out of room %d\n", 
+                    occupants[i + j].id, occupants[i + j].room);
+            opal_hotel_checkout(&hotel, occupants[i + j].room);
+            --num_occupied;
+        }
+        for (j=0; j < 30; j++) {
+            if (OPAL_SUCCESS != 
+                opal_hotel_checkin(&hotel, (void*) &(occupants[i + j]), &rm)) {
+                fprintf(stderr, "Hotel is fully occupied\n");
+                continue;
+            }
+            occupants[i + j].room = rm;
+            fprintf(stderr, "Occupant %d checked into room %d\n", 
+                    occupants[i + j].id, rm);
+            ++num_occupied;
+        }
+        fprintf(stderr, "---------------------------------------\n");
+    }
+
+    /* sit here and see if we get an eviction notice */
+    fprintf(stderr, "Waiting for %d evictions...\n", num_occupied);
+    while (num_evicted < num_occupied) {
+        opal_progress();
+    }
+    fprintf(stderr, "All occupants evicted!\n");
+
+    OBJ_DESTRUCT(&hotel);
+
+    opal_finalize();
+    return 0;
+}