It's not about how much memory we use, but about how we use it.

Keeping the cache misses as low as possible is always a good approach.
The opal_list_t is widely used, it should be a highly optimized class.
The same functionality can be reached with one one sentinel instead
of 2 currently used.
I don't have anything against the STL version, but so far nothing can
compare with the Knuth algorithm. I replace the current implementation
with a modified version of the Knuth algorithm (the one described in
The Art of Computer Programming). As expected, the latency went down.

This commit was SVN r10776.

ompi/class/ompi_seq_tracker.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -95,19 +95,15 @@ void ompi_seq_tracker_insert(ompi_seq_tracker_t* seq_tracker,
     ompi_seq_tracker_range_t* item = seq_tracker->seq_ids_current;
     int8_t direction = 0; /* 1 is next, -1 is previous */
     ompi_seq_tracker_range_t *new_item, *next_item, *prev_item;
-    while(true) { 
-        if( item == NULL || item == (ompi_seq_tracker_range_t*) &seq_ids->opal_list_tail )  { 
+    ompi_seq_tracker_range_t* sentinel = (ompi_seq_tracker_range_t*)&seq_ids->opal_list_sentinel;
+
+    while( true ) { 
+        if( item == NULL || item == sentinel )  { 
             new_item = OBJ_NEW(ompi_seq_tracker_range_t);
             new_item->seq_id_low = new_item->seq_id_high = seq_id; 
             opal_list_append(seq_ids, (opal_list_item_t*) new_item);
             seq_tracker->seq_ids_current = (ompi_seq_tracker_range_t*) new_item;
             return;
-        } else if( item == (ompi_seq_tracker_range_t*) &seq_ids->opal_list_head ) { 
-            new_item = OBJ_NEW(ompi_seq_tracker_range_t);
-            new_item->seq_id_low = new_item->seq_id_high = seq_id; 
-            opal_list_prepend(seq_ids, (opal_list_item_t*) new_item); 
-            seq_tracker->seq_ids_current = (ompi_seq_tracker_range_t*) new_item;
-            return; 
 
         } else if(item->seq_id_high >= seq_id && item->seq_id_low <= seq_id ) { 
 

opal/class/opal_list.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -83,18 +83,13 @@ static void opal_list_construct(opal_list_t *list)
        should never be removed from this list, added to another list,
        etc.  So set them to sentinel values. */
 
-    OBJ_CONSTRUCT( &(list->opal_list_head), opal_list_item_t );
-    list->opal_list_head.opal_list_item_refcount  = 1;
-    list->opal_list_head.opal_list_item_belong_to = list;
-    OBJ_CONSTRUCT( &(list->opal_list_tail), opal_list_item_t );
-    list->opal_list_tail.opal_list_item_refcount  = 1;
-    list->opal_list_tail.opal_list_item_belong_to = list;
+    OBJ_CONSTRUCT( &(list->opal_list_sentinel), opal_list_item_t );
+    list->opal_list_sentinel.opal_list_item_refcount  = 1;
+    list->opal_list_sentinel.opal_list_item_belong_to = list;
 #endif
 
-    list->opal_list_head.opal_list_prev = NULL;
-    list->opal_list_head.opal_list_next = &list->opal_list_tail;
-    list->opal_list_tail.opal_list_prev = &list->opal_list_head;
-    list->opal_list_tail.opal_list_next = NULL;
+    list->opal_list_sentinel.opal_list_next = &list->opal_list_sentinel;
+    list->opal_list_sentinel.opal_list_prev = &list->opal_list_sentinel;
     list->opal_list_length = 0;
 }
 
@@ -125,9 +120,7 @@ bool opal_list_insert(opal_list_t *list, opal_list_item_t *item, long long idx)
     if ( 0 == idx )
     {
         opal_list_prepend(list, item);
-    }
-    else
-    {
+    } else {
 #if OMPI_ENABLE_DEBUG
         /* Spot check: ensure that this item is previously on no
            lists */
@@ -135,7 +128,7 @@ bool opal_list_insert(opal_list_t *list, opal_list_item_t *item, long long idx)
         assert(0 == item->opal_list_item_refcount);
 #endif
         /* pointer to element 0 */
-        ptr = list->opal_list_head.opal_list_next;
+        ptr = list->opal_list_sentinel.opal_list_next;
         for ( i = 0; i < idx-1; i++ )
             ptr = ptr->opal_list_next;
 

opal/class/opal_list.h

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -146,10 +146,8 @@ struct opal_list_t
 {
     opal_object_t       super;
     /**< Generic parent class for all Open MPI objects */
-    opal_list_item_t    opal_list_head;
-    /**< Head item of the list */
-    opal_list_item_t    opal_list_tail;
-    /**< Tail item of the list */
+    opal_list_item_t    opal_list_sentinel;
+    /**< Head and tail item of the list */
     volatile size_t     opal_list_length;
     /**< Quick reference to the number of items in the list */
 };
@@ -173,8 +171,8 @@ typedef struct opal_list_t opal_list_t;
  */
 static inline bool opal_list_is_empty(opal_list_t* list)
 {
-    return (list->opal_list_head.opal_list_next == 
-            &(list->opal_list_tail));
+    return (list->opal_list_sentinel.opal_list_next == 
+            &(list->opal_list_sentinel));
 }
 
 
@@ -194,7 +192,7 @@ static inline bool opal_list_is_empty(opal_list_t* list)
  */
 static inline opal_list_item_t* opal_list_get_first(opal_list_t* list)
 {
-    opal_list_item_t* item = (opal_list_item_t *)list->opal_list_head.opal_list_next;
+    opal_list_item_t* item = (opal_list_item_t*)list->opal_list_sentinel.opal_list_next;
 #if OMPI_ENABLE_DEBUG
     /* Spot check: ensure that the first item is only on one list */
 
@@ -221,7 +219,7 @@ static inline opal_list_item_t* opal_list_get_first(opal_list_t* list)
  */
 static inline opal_list_item_t* opal_list_get_last(opal_list_t* list)
 {
-    opal_list_item_t* item = (opal_list_item_t *)list->opal_list_tail.opal_list_prev;
+    opal_list_item_t* item = (opal_list_item_t *)list->opal_list_sentinel.opal_list_prev;
 #if OMPI_ENABLE_DEBUG
     /* Spot check: ensure that the last item is only on one list */
 
@@ -251,7 +249,7 @@ static inline opal_list_item_t* opal_list_get_last(opal_list_t* list)
  */
 static inline opal_list_item_t* opal_list_get_begin(opal_list_t* list)
 {
-    return &(list->opal_list_head);
+    return &(list->opal_list_sentinel);
 }
 
 /**
@@ -273,7 +271,7 @@ static inline opal_list_item_t* opal_list_get_begin(opal_list_t* list)
  */
 static inline opal_list_item_t* opal_list_get_end(opal_list_t* list)
 {
-    return &(list->opal_list_tail);
+    return &(list->opal_list_sentinel);
 }
 
 
@@ -417,6 +415,7 @@ static inline void _opal_list_append(opal_list_t *list, opal_list_item_t *item
 #endif  /* OMPI_ENABLE_DEBUG */
                                      )
 {
+    opal_list_item_t* sentinel = &(list->opal_list_sentinel);
 #if OMPI_ENABLE_DEBUG
   /* Spot check: ensure that this item is previously on no lists */
 
@@ -427,16 +426,16 @@ static inline void _opal_list_append(opal_list_t *list, opal_list_item_t *item
 #endif
 
   /* set new element's previous pointer */
-  item->opal_list_prev=list->opal_list_tail.opal_list_prev;
+  item->opal_list_prev = sentinel->opal_list_prev;
 
   /* reset previous pointer on current last element */
-  list->opal_list_tail.opal_list_prev->opal_list_next=item;
+  sentinel->opal_list_prev->opal_list_next = item;
 
   /* reset new element's next pointer */
-  item->opal_list_next=&(list->opal_list_tail);
+  item->opal_list_next = sentinel;
 
   /* reset the list's tail element previous pointer */
-  list->opal_list_tail.opal_list_prev = item;
+  sentinel->opal_list_prev = item;
 
   /* increment list element counter */
   list->opal_list_length++;
@@ -468,6 +467,7 @@ static inline void _opal_list_append(opal_list_t *list, opal_list_item_t *item
 static inline void opal_list_prepend(opal_list_t *list, 
                                      opal_list_item_t *item) 
 {
+    opal_list_item_t* sentinel = &(list->opal_list_sentinel);
 #if OMPI_ENABLE_DEBUG
   /* Spot check: ensure that this item is previously on no lists */
 
@@ -476,16 +476,16 @@ static inline void opal_list_prepend(opal_list_t *list,
 #endif
 
   /* reset item's next pointer */
-  item->opal_list_next = list->opal_list_head.opal_list_next;
+  item->opal_list_next = sentinel->opal_list_next;
   
   /* reset item's previous pointer */
-  item->opal_list_prev = &(list->opal_list_head);
+  item->opal_list_prev = sentinel;
   
   /* reset previous first element's previous poiner */
-  list->opal_list_head.opal_list_next->opal_list_prev = item;
+  sentinel->opal_list_next->opal_list_prev = item;
   
   /* reset head's next pointer */
-  list->opal_list_head.opal_list_next = item;
+  sentinel->opal_list_next = item;
   
   /* increment list element counter */
   list->opal_list_length++;
@@ -531,20 +531,20 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
 #if OMPI_ENABLE_DEBUG
   /* Spot check: ensure that the first item is only on this list */
 
-  assert(1 == list->opal_list_head.opal_list_next->opal_list_item_refcount);
+  assert(1 == list->opal_list_sentinel.opal_list_next->opal_list_item_refcount);
 #endif
 
   /* reset list length counter */
   list->opal_list_length--;
   
   /* get pointer to first element on the list */
-  item = list->opal_list_head.opal_list_next;
+  item = list->opal_list_sentinel.opal_list_next;
   
   /* reset previous pointer of next item on the list */
-  item->opal_list_next->opal_list_prev=item->opal_list_prev;
+  item->opal_list_next->opal_list_prev = item->opal_list_prev;
   
   /* reset the head next pointer */
-  list->opal_list_head.opal_list_next=item->opal_list_next;
+  list->opal_list_sentinel.opal_list_next = item->opal_list_next;
   
 #if OMPI_ENABLE_DEBUG
   assert( list == item->opal_list_item_belong_to );
@@ -587,26 +587,26 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
   */
   volatile opal_list_item_t  *item;
   if ( 0 == list->opal_list_length ) {
-    return (opal_list_item_t *)NULL;
+      return (opal_list_item_t *)NULL;
   }
   
 #if OMPI_ENABLE_DEBUG
   /* Spot check: ensure that the first item is only on this list */
 
-  assert(1 == list->opal_list_tail.opal_list_prev->opal_list_item_refcount);
+  assert(1 == list->opal_list_sentinel.opal_list_prev->opal_list_item_refcount);
 #endif
 
   /* reset list length counter */
   list->opal_list_length--;
   
   /* get item */
-  item = list->opal_list_tail.opal_list_prev;
+  item = list->opal_list_sentinel.opal_list_prev;
   
   /* reset previous pointer on next to last pointer */
-  item->opal_list_prev->opal_list_next=item->opal_list_next;
+  item->opal_list_prev->opal_list_next = item->opal_list_next;
   
   /* reset tail's previous pointer */
-  list->opal_list_tail.opal_list_prev=item->opal_list_prev;
+  list->opal_list_sentinel.opal_list_prev = item->opal_list_prev;
   
 #if OMPI_ENABLE_DEBUG
   assert( list == item->opal_list_item_belong_to );