Remove include/opal/sys/cache.h -- its only purpose in life was to

#define CACHE_LINE_SIZE to 128.  This name has a conflict on NetBSD,
and it seems kinda odd to have a header file that ''only'' defines a
single value.  Also, we'll soon be raising hwloc to be a first-class
item, so having this file around seemed kinda weird.

Therefore, I replaced CACHE_LINE_SIZE with opal_cache_line_size, an
int (in opal/runtime/opal_init.c and opal/runtime/opal.h) on the
rationale that we can fill this in at runtime with hwloc info (trunk
and v1.5/beyond, only).  The only place we ''needed'' a compile-time
CACHE_LINE_SIZE was in the BTL SM (for struct padding), so I made a
new BTL_SM_ preprocessor macro with the old CACHE_LINE_SIZE value
(128).  That use isn't suitable for run-time hwloc information,
anyway.

This commit was SVN r23349.

ompi/class/ompi_free_list.c

@@ -11,6 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -23,7 +24,6 @@
 #include "ompi/class/ompi_free_list.h"
 #include "opal/align.h"
 #include "opal/util/output.h"
-#include "opal/sys/cache.h"
 #include "ompi/mca/mpool/mpool.h"
 
 static void ompi_free_list_construct(ompi_free_list_t* fl);

ompi/class/ompi_free_list.h

@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -25,7 +26,7 @@
 #include "opal/prefetch.h"
 #include "opal/threads/condition.h"
 #include "ompi/constants.h"
-#include "opal/sys/cache.h"
+#include "opal/runtime/opal.h"
 
 BEGIN_C_DECLS
 
@@ -101,7 +102,7 @@ static inline int ompi_free_list_init(
     int num_elements_per_alloc,
     struct mca_mpool_base_module_t* mpool)
 {
-    return ompi_free_list_init_ex(free_list, element_size, CACHE_LINE_SIZE,
+    return ompi_free_list_init_ex(free_list, element_size, opal_cache_line_size,
             element_class, num_elements_to_alloc, max_elements_to_alloc,
                                   num_elements_per_alloc, mpool, NULL, NULL);
 }

ompi/class/ompi_rb_tree.c

@@ -51,8 +51,8 @@ void ompi_rb_tree_construct(opal_object_t * object)
     tree->root_ptr = NULL;
     OBJ_CONSTRUCT(&(tree->free_list), ompi_free_list_t);
     ompi_free_list_init_new(&(tree->free_list), sizeof(ompi_rb_tree_node_t),
-            CACHE_LINE_SIZE, OBJ_CLASS(ompi_rb_tree_node_t),
+            opal_cache_line_size, OBJ_CLASS(ompi_rb_tree_node_t),
-            0,CACHE_LINE_SIZE,
+            0,opal_cache_line_size,
             0, -1 , 128, NULL);
 }
 

ompi/class/ompi_seq_tracker.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -19,7 +20,6 @@
 #include "ompi_config.h"
 
 #include "ompi/class/ompi_seq_tracker.h"
-#include "opal/sys/cache.h"
 
 
 

ompi/mca/allocator/basic/allocator_basic.c

@@ -96,9 +96,9 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
  
     ompi_free_list_init_new(&module->seg_descriptors,
         sizeof(mca_allocator_basic_segment_t),
-        CACHE_LINE_SIZE,
+        opal_cache_line_size,
         OBJ_CLASS(mca_allocator_basic_segment_t),
-        0,CACHE_LINE_SIZE,
+        0,opal_cache_line_size,
         0,  /* initial size */
         -1, /* maximum size */
         16, /* increment to grow by */

ompi/mca/bml/r2/bml_r2_component.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -17,7 +18,6 @@
  */
 
 #include "ompi_config.h"
-#include "opal/sys/cache.h"
 #include "opal/event/event.h"
 #include "mpi.h"
 #include "ompi/mca/bml/bml.h"

ompi/mca/btl/elan/btl_elan_component.c

@@ -170,9 +170,9 @@ mca_btl_elan_component_init( int *num_btl_modules,
     OBJ_CONSTRUCT(&mca_btl_elan_component.elan_procs, opal_list_t);
     ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_eager,
                              sizeof(mca_btl_elan_frag_t) + mca_btl_elan_module.super.btl_eager_limit,
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(mca_btl_elan_frag_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              mca_btl_elan_component.elan_free_list_num,
                              mca_btl_elan_component.elan_free_list_max,
                              mca_btl_elan_component.elan_free_list_inc,
@@ -180,9 +180,9 @@ mca_btl_elan_component_init( int *num_btl_modules,
 
     ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_user,
                              sizeof(mca_btl_elan_frag_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(mca_btl_elan_frag_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              mca_btl_elan_component.elan_free_list_num,
                              mca_btl_elan_component.elan_free_list_max,
                              mca_btl_elan_component.elan_free_list_inc,
@@ -190,9 +190,9 @@ mca_btl_elan_component_init( int *num_btl_modules,
 
     ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_max,
                              sizeof(mca_btl_elan_frag_t)+mca_btl_elan_module.super.btl_max_send_size,
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(mca_btl_elan_frag_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              mca_btl_elan_component.elan_free_list_num,
                              mca_btl_elan_component.elan_free_list_max,
                              mca_btl_elan_component.elan_free_list_inc,

ompi/mca/btl/gm/btl_gm_component.c

@@ -273,10 +273,10 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
     /* initialize free lists */
     ompi_free_list_init_new( &btl->gm_frag_eager,
             sizeof (mca_btl_gm_frag_eager_t),
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS (mca_btl_gm_frag_eager_t),
             (1 << mca_btl_gm_component.gm_eager_frag_size) + sizeof (uintptr_t),
-            CACHE_LINE_SIZE, 
+            opal_cache_line_size, 
             btl->gm_max_send_tokens,  
             mca_btl_gm_component.gm_free_list_max, 
             mca_btl_gm_component.gm_free_list_inc,
@@ -284,10 +284,10 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
 
     ompi_free_list_init_new( &btl->gm_frag_max,
             sizeof (mca_btl_gm_frag_max_t),
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS (mca_btl_gm_frag_max_t),
             (1 << mca_btl_gm_component.gm_max_frag_size) + sizeof (uintptr_t),
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             btl->gm_max_recv_tokens,
             mca_btl_gm_component.gm_free_list_max, 
             mca_btl_gm_component.gm_free_list_inc,
@@ -295,9 +295,9 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
 
     ompi_free_list_init_new( &btl->gm_frag_user,
             sizeof (mca_btl_gm_frag_user_t),
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS (mca_btl_gm_frag_user_t),
-            0,CACHE_LINE_SIZE,
+            0,opal_cache_line_size,
             mca_btl_gm_component.gm_free_list_num,  
             mca_btl_gm_component.gm_free_list_max, 
             mca_btl_gm_component.gm_free_list_inc,

ompi/mca/btl/mx/btl_mx_component.c

@@ -467,9 +467,9 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
 
     ompi_free_list_init_new( &mca_btl_mx_component.mx_send_eager_frags,
                          sizeof(mca_btl_mx_frag_t) + mca_btl_mx_module.super.btl_eager_limit,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_btl_mx_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_mx_component.mx_free_list_num,
                          mca_btl_mx_component.mx_free_list_max,
                          mca_btl_mx_component.mx_free_list_inc,
@@ -477,9 +477,9 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
 
     ompi_free_list_init_new( &mca_btl_mx_component.mx_send_user_frags,
                          sizeof(mca_btl_mx_frag_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_btl_mx_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_mx_component.mx_free_list_num,
                          mca_btl_mx_component.mx_free_list_max,
                          mca_btl_mx_component.mx_free_list_inc,

ompi/mca/btl/ofud/btl_ofud.c

@@ -597,9 +597,9 @@ int mca_btl_ud_module_init(mca_btl_ud_module_t *ud_btl)
 
     ompi_free_list_init_new(&ud_btl->recv_frags,
                         length + sizeof(mca_btl_ud_ib_header_t),
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_ud_recv_frag_t),
-                        length_payload,CACHE_LINE_SIZE,
+                        length_payload,opal_cache_line_size,
                         mca_btl_ofud_component.rd_num,
                         mca_btl_ofud_component.rd_num,
                         mca_btl_ofud_component.rd_num,
@@ -607,9 +607,9 @@ int mca_btl_ud_module_init(mca_btl_ud_module_t *ud_btl)
 #if 0
     ompi_free_list_init_new(&ud_btl->recv_frags,
                         length + sizeof(mca_btl_ud_ib_header_t),
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_ud_recv_frag_t),
-                        length_payload,CACHE_LINE_SIZE,
+                        length_payload,opal_cache_line_size,
                          mca_btl_ofud_component.rd_num_init,
                          mca_btl_ofud_component.rd_num_max,
                          mca_btl_ofud_component.rd_num_inc,
@@ -644,9 +644,9 @@ int mca_btl_ud_module_init(mca_btl_ud_module_t *ud_btl)
 
     ompi_free_list_init_new(&ud_btl->send_frags,
                         length,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_ud_send_frag_t),
-                        length_payload,CACHE_LINE_SIZE,
+                        length_payload,opal_cache_line_size,
                         mca_btl_ofud_component.sd_num >> 1,
                         -1,
                         mca_btl_ofud_component.sd_num << 2,
@@ -662,9 +662,9 @@ int mca_btl_ud_module_init(mca_btl_ud_module_t *ud_btl)
 
     ompi_free_list_init_new(&ud_btl->user_frags,
                         length,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_ud_user_frag_t),
-                        length_payload,CACHE_LINE_SIZE,
+                        length_payload,opal_cache_line_size,
                         mca_btl_ofud_component.sd_num >> 1,
                         -1,
                         mca_btl_ofud_component.sd_num << 2,

ompi/mca/btl/openib/btl_openib_component.c

@@ -1026,7 +1026,7 @@ static int prepare_device_for_use(mca_btl_openib_device_t *device)
     init_data->list = &device->send_free_control;
 
     rc = ompi_free_list_init_ex_new(&device->send_free_control,
-                sizeof(mca_btl_openib_send_control_frag_t), CACHE_LINE_SIZE,
+                sizeof(mca_btl_openib_send_control_frag_t), opal_cache_line_size,
                 OBJ_CLASS(mca_btl_openib_send_control_frag_t), length,
                 mca_btl_openib_component.buffer_alignment,
                 mca_btl_openib_component.ib_free_list_num, -1,
@@ -1060,7 +1060,7 @@ static int prepare_device_for_use(mca_btl_openib_device_t *device)
         init_data->list = &device->qps[qp].send_free;
 
         rc = ompi_free_list_init_ex_new(init_data->list,
-                    sizeof(mca_btl_openib_send_frag_t), CACHE_LINE_SIZE,
+                    sizeof(mca_btl_openib_send_frag_t), opal_cache_line_size,
                     OBJ_CLASS(mca_btl_openib_send_frag_t), length,
                     mca_btl_openib_component.buffer_alignment,
                     mca_btl_openib_component.ib_free_list_num,
@@ -1093,7 +1093,7 @@ static int prepare_device_for_use(mca_btl_openib_device_t *device)
         init_data->list = &device->qps[qp].recv_free;
 
         if(OMPI_SUCCESS != ompi_free_list_init_ex_new(init_data->list,
-                    sizeof(mca_btl_openib_recv_frag_t), CACHE_LINE_SIZE,
+                    sizeof(mca_btl_openib_recv_frag_t), opal_cache_line_size,
                     OBJ_CLASS(mca_btl_openib_recv_frag_t),
                     length, mca_btl_openib_component.buffer_alignment,
                     mca_btl_openib_component.ib_free_list_num,

ompi/mca/btl/portals/btl_portals_component.c

@@ -257,9 +257,9 @@ mca_btl_portals_component_init(int *num_btls,
     ompi_free_list_init_new(&(mca_btl_portals_module.portals_frag_eager),
                         sizeof(mca_btl_portals_frag_eager_t) + 
                         mca_btl_portals_module.super.btl_eager_limit,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_portals_frag_eager_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         mca_btl_portals_component.portals_free_list_init_num,
                         mca_btl_portals_component.portals_free_list_eager_max_num,
                         mca_btl_portals_component.portals_free_list_inc_num,
@@ -269,9 +269,9 @@ mca_btl_portals_component_init(int *num_btls,
     ompi_free_list_init_new(&(mca_btl_portals_module.portals_frag_max),
                         sizeof(mca_btl_portals_frag_max_t) + 
                         mca_btl_portals_module.super.btl_max_send_size,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_portals_frag_max_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         mca_btl_portals_component.portals_free_list_init_num,
                         mca_btl_portals_component.portals_free_list_max_num,
                         mca_btl_portals_component.portals_free_list_inc_num,
@@ -280,9 +280,9 @@ mca_btl_portals_component_init(int *num_btls,
     /* user frags */
     ompi_free_list_init_new(&(mca_btl_portals_module.portals_frag_user),
                         sizeof(mca_btl_portals_frag_user_t),
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_btl_portals_frag_user_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         mca_btl_portals_component.portals_free_list_init_num,
                         mca_btl_portals_component.portals_free_list_max_num,
                         mca_btl_portals_component.portals_free_list_inc_num,

ompi/mca/btl/self/btl_self_component.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
 #endif  /* HAVE_SYS_TYPES_H */
 
 #include "ompi/constants.h"
-#include "opal/sys/cache.h"
+#include "opal/runtime/opal.h"
 #include "opal/event/event.h"
 #include "opal/mca/base/mca_base_param.h"
 #include "btl_self.h"
@@ -137,27 +138,27 @@ mca_btl_base_module_t** mca_btl_self_component_init( int *num_btls,
     /* initialize free lists */
     ompi_free_list_init_new( &mca_btl_self_component.self_frags_eager, 
                          sizeof(mca_btl_self_frag_eager_t) + mca_btl_self.btl_eager_limit,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_btl_self_frag_eager_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_self_component.free_list_num,
                          mca_btl_self_component.free_list_max,
                          mca_btl_self_component.free_list_inc,
                          NULL );
     ompi_free_list_init_new( &mca_btl_self_component.self_frags_send, 
                          sizeof(mca_btl_self_frag_send_t) + mca_btl_self.btl_max_send_size,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_btl_self_frag_send_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_self_component.free_list_num,
                          mca_btl_self_component.free_list_max,
                          mca_btl_self_component.free_list_inc,
                          NULL );
     ompi_free_list_init_new( &mca_btl_self_component.self_frags_rdma, 
                          sizeof(mca_btl_self_frag_rdma_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_btl_self_frag_rdma_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_self_component.free_list_num,
                          mca_btl_self_component.free_list_max,
                          mca_btl_self_component.free_list_inc,

ompi/mca/btl/sm/btl_sm.c

@@ -115,7 +115,7 @@ static void *mpool_calloc(size_t nmemb, size_t size)
     size_t bsize = nmemb * size;
     mca_mpool_base_module_t *mpool = mca_btl_sm_component.sm_mpool;
 
-    buf = mpool->mpool_alloc(mpool, bsize, CACHE_LINE_SIZE, 0, NULL);
+    buf = mpool->mpool_alloc(mpool, bsize, opal_cache_line_size, 0, NULL);
 
     if (NULL == buf)
         return NULL;
@@ -220,14 +220,14 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
          * - eager fragments (2*n of them, allocated in sm_free_list_inc chunks)
          * - max fragments (sm_free_list_num of them)
          *
-         * On top of all that, we sprinkle in some number of "CACHE_LINE_SIZE"
+         * On top of all that, we sprinkle in some number of "opal_cache_line_size"
          * additions to account for some padding and edge effects that may lie
          * in the allocator.
          */
         res.size =
-            FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * CACHE_LINE_SIZE )
+            FIFO_MAP_NUM(n) * ( sizeof(sm_fifo_t) + sizeof(void *) * m->fifo_size + 4 * opal_cache_line_size )
-            + ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit   + 2 * CACHE_LINE_SIZE )
+            + ( 2 * n + m->sm_free_list_inc ) * ( m->eager_limit   + 2 * opal_cache_line_size )
-            +           m->sm_free_list_num   * ( m->max_frag_size + 2 * CACHE_LINE_SIZE );
+            +           m->sm_free_list_num   * ( m->max_frag_size + 2 * opal_cache_line_size );
 
         /* before we multiply by n, make sure the result won't overflow */
         /* Stick that little pad in, particularly since we'll eventually
@@ -273,12 +273,12 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
     /* Pass in a data segment alignment of 0 to get no data
        segment (only the shared control structure) */
     size = sizeof(mca_common_sm_seg_header_t) +
-        n * (sizeof(sm_fifo_t*) + sizeof(char *) + sizeof(uint16_t)) + CACHE_LINE_SIZE;
+        n * (sizeof(sm_fifo_t*) + sizeof(char *) + sizeof(uint16_t)) + opal_cache_line_size;
     procs = ompi_proc_world(&num_procs);
     if (!(mca_btl_sm_component.sm_seg =
           mca_common_sm_init(procs, num_procs, size, sm_ctl_file,
                              sizeof(mca_common_sm_seg_header_t),
-                             CACHE_LINE_SIZE))) {
+                             opal_cache_line_size))) {
         opal_output(0, "mca_btl_sm_add_procs: unable to create shared memory "
                     "BTL coordinating strucure :: size %lu \n",
                     (unsigned long)size);
@@ -338,8 +338,8 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
     length_payload =
         sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.eager_limit;
     i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_eager, length,
-                                CACHE_LINE_SIZE, OBJ_CLASS(mca_btl_sm_frag1_t),
+                                opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag1_t),
-                                length_payload, CACHE_LINE_SIZE,
+                                length_payload, opal_cache_line_size,
                                 mca_btl_sm_component.sm_free_list_num,
                                 mca_btl_sm_component.sm_free_list_max,
                                 mca_btl_sm_component.sm_free_list_inc,
@@ -351,8 +351,8 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
     length_payload =
         sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.max_frag_size;
     i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_max, length,
-                                CACHE_LINE_SIZE, OBJ_CLASS(mca_btl_sm_frag2_t),
+                                opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag2_t),
-                                length_payload, CACHE_LINE_SIZE,
+                                length_payload, opal_cache_line_size,
                                 mca_btl_sm_component.sm_free_list_num,
                                 mca_btl_sm_component.sm_free_list_max,
                                 mca_btl_sm_component.sm_free_list_inc,
@@ -362,8 +362,8 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
 
     i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_user, 
 		    sizeof(mca_btl_sm_user_t),
-		    CACHE_LINE_SIZE, OBJ_CLASS(mca_btl_sm_user_t),
+		    opal_cache_line_size, OBJ_CLASS(mca_btl_sm_user_t),
-		    sizeof(mca_btl_sm_hdr_t), CACHE_LINE_SIZE,
+		    sizeof(mca_btl_sm_hdr_t), opal_cache_line_size,
 		    mca_btl_sm_component.sm_free_list_num,
 		    mca_btl_sm_component.sm_free_list_max,
 		    mca_btl_sm_component.sm_free_list_inc,

ompi/mca/btl/sm/btl_sm.h

@@ -101,24 +101,35 @@ BEGIN_C_DECLS
  */
 
 #define SM_FIFO_FREE  (void *) (-2)
+/* We can't use opal_cache_line_size here because we need a
+   compile-time constant for padding the struct.  We can't really have
+   a compile-time constant that is portable, either (e.g., compile on
+   one machine and run on another).  So just use a big enough cache
+   line that should hopefully be good in most places. */
+#define SM_CACHE_LINE_PAD 128
 
 struct sm_fifo_t {
     /* This queue pointer is used only by the heads. */
-    volatile void **queue;           char pad0[CACHE_LINE_SIZE - sizeof(void **)           ];
+    volatile void **queue;           
+    char pad0[SM_CACHE_LINE_PAD - sizeof(void **)];
     /* This lock is used by the heads. */
-    opal_atomic_lock_t head_lock;    char pad1[CACHE_LINE_SIZE - sizeof(opal_atomic_lock_t)];
+    opal_atomic_lock_t head_lock;    
+    char pad1[SM_CACHE_LINE_PAD - sizeof(opal_atomic_lock_t)];
     /* This index is used by the head holding the head lock. */
-    volatile int head;               char pad2[CACHE_LINE_SIZE - sizeof(int)               ];
+    volatile int head;               
+    char pad2[SM_CACHE_LINE_PAD - sizeof(int)];
     /* This mask is used "read only" by all processes. */
-    unsigned int mask;               char pad3[CACHE_LINE_SIZE - sizeof(int)               ];
+    unsigned int mask;               
+    char pad3[SM_CACHE_LINE_PAD - sizeof(int)];
     /* The following are used only by the tail. */
     volatile void **queue_recv;
     opal_atomic_lock_t tail_lock;
     volatile int tail;
     int num_to_clear;
-    int lazy_free;                   char pad4[CACHE_LINE_SIZE - sizeof(void **)
+    int lazy_free;                   
-                                                               - sizeof(opal_atomic_lock_t)
+    char pad4[SM_CACHE_LINE_PAD - sizeof(void **) -
-                                                               - sizeof(int) * 3           ];
+              sizeof(opal_atomic_lock_t) -
+              sizeof(int) * 3];
 };
 typedef struct sm_fifo_t sm_fifo_t;
 
@@ -289,7 +300,7 @@ static inline int sm_fifo_init(int fifo_size, mca_mpool_base_module_t *mpool,
 
     /* allocate the queue in the receiver's address space */
     fifo->queue_recv = (volatile void **)mpool->mpool_alloc(
-            mpool, sizeof(void *) * qsize, CACHE_LINE_SIZE, 0, NULL);
+            mpool, sizeof(void *) * qsize, opal_cache_line_size, 0, NULL);
     if(NULL == fifo->queue_recv) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }

ompi/mca/btl/sm/btl_sm_component.c

@@ -41,7 +41,6 @@
 #endif  /* HAVE_SYS_STAT_H */
 
 #include "ompi/constants.h"
-#include "opal/sys/cache.h"
 #include "opal/event/event.h"
 #include "opal/util/output.h"
 #include "orte/util/proc_info.h"

ompi/mca/btl/tcp/btl_tcp_component.c

@@ -920,9 +920,9 @@ mca_btl_base_module_t** mca_btl_tcp_component_init(int *num_btl_modules,
     ompi_free_list_init_new( &mca_btl_tcp_component.tcp_frag_eager,
                          sizeof (mca_btl_tcp_frag_eager_t) + 
                          mca_btl_tcp_module.super.btl_eager_limit,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS (mca_btl_tcp_frag_eager_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_tcp_component.tcp_free_list_num,
                          mca_btl_tcp_component.tcp_free_list_max,
                          mca_btl_tcp_component.tcp_free_list_inc,
@@ -931,9 +931,9 @@ mca_btl_base_module_t** mca_btl_tcp_component_init(int *num_btl_modules,
     ompi_free_list_init_new( &mca_btl_tcp_component.tcp_frag_max,
                          sizeof (mca_btl_tcp_frag_max_t) + 
                          mca_btl_tcp_module.super.btl_max_send_size,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS (mca_btl_tcp_frag_max_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_tcp_component.tcp_free_list_num,
                          mca_btl_tcp_component.tcp_free_list_max,
                          mca_btl_tcp_component.tcp_free_list_inc,
@@ -941,9 +941,9 @@ mca_btl_base_module_t** mca_btl_tcp_component_init(int *num_btl_modules,
                                                                                                                                   
     ompi_free_list_init_new( &mca_btl_tcp_component.tcp_frag_user,
                          sizeof (mca_btl_tcp_frag_user_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS (mca_btl_tcp_frag_user_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_btl_tcp_component.tcp_free_list_num,
                          mca_btl_tcp_component.tcp_free_list_max,
                          mca_btl_tcp_component.tcp_free_list_inc,

ompi/mca/coll/sm2/coll_sm2.h

@@ -260,7 +260,7 @@ BEGIN_C_DECLS
 
         /* pading */
         /* Note: need to change this so it takes less memory */
-        char padding[2*CACHE_LINE_SIZE-2*sizeof(long long)];
+        char padding[2*opal_cache_line_size-2*sizeof(long long)];
     };
 
     typedef struct mca_coll_sm2_nb_request_process_shared_mem_t

ompi/mca/coll/sm2/coll_sm2_component.c

@@ -140,7 +140,7 @@ static int sm2_open(void)
 
     /* Data region alignment (bytes) - per proc */
     cs->sm2_data_alignment=
-        mca_coll_sm2_param_register_int("sm2_data_alignment",CACHE_LINE_SIZE);
+        mca_coll_sm2_param_register_int("sm2_data_alignment",opal_cache_line_size);
 
     /* Number of memory banks */
     cs->sm2_num_mem_banks=

ompi/mca/coll/sm2/coll_sm2_module.c

@@ -630,7 +630,7 @@ static int init_sm2_barrier(struct ompi_communicator_t *comm,
                 (mca_coll_sm2_nb_request_process_shared_mem_t *)
                 (module->shared_memory_region + 
                  /* there are 2 barrier structs per bank */
-                 (2*i+j)*CACHE_LINE_SIZE);
+                 (2*i+j)*opal_cache_line_size);
             /* initialize per-process flags */
             for(k=0 ; k < comm_size ; k++ ) {
                 sm_address=(mca_coll_sm2_nb_request_process_shared_mem_t *)
@@ -943,7 +943,7 @@ mca_coll_sm2_comm_query(struct ompi_communicator_t *comm, int *priority)
      *   manage the memory resources.
      */
     /* for each bank, 2 sets of barrier buffers */
-    mem_management_per_proc_per_block= 2 * CACHE_LINE_SIZE ;
+    mem_management_per_proc_per_block= 2 * opal_cache_line_size ;
     /* add in number of banks */
     mem_management_per_proc= mem_management_per_proc_per_block *
         mca_coll_sm2_component.sm2_num_mem_banks;

ompi/mca/crcp/bkmrk/crcp_bkmrk_pml.c

@@ -1101,9 +1101,9 @@ int ompi_crcp_bkmrk_pml_init(void) {
     OBJ_CONSTRUCT(&coord_state_free_list, ompi_free_list_t);
     ompi_free_list_init_new( &coord_state_free_list,
                              sizeof(ompi_crcp_bkmrk_pml_state_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_bkmrk_pml_state_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              4,  /* Initial number */
                              -1, /* Max = Unlimited */
                              4,  /* Increment by */
@@ -1112,9 +1112,9 @@ int ompi_crcp_bkmrk_pml_init(void) {
     OBJ_CONSTRUCT(&content_ref_free_list, ompi_free_list_t);
     ompi_free_list_init_new( &content_ref_free_list,
                              sizeof(ompi_crcp_bkmrk_pml_message_content_ref_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_bkmrk_pml_message_content_ref_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              80, /* Initial number */
                              -1, /* Max = Unlimited */
                              32, /* Increment by */
@@ -1123,9 +1123,9 @@ int ompi_crcp_bkmrk_pml_init(void) {
     OBJ_CONSTRUCT(&peer_ref_free_list, ompi_free_list_t);
     ompi_free_list_init_new( &peer_ref_free_list,
                              sizeof(ompi_crcp_bkmrk_pml_peer_ref_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_bkmrk_pml_peer_ref_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              16, /* Initial number */
                              -1, /* Max = Unlimited */
                              16, /* Increment by */
@@ -1134,9 +1134,9 @@ int ompi_crcp_bkmrk_pml_init(void) {
     OBJ_CONSTRUCT(&traffic_msg_ref_free_list, ompi_free_list_t);
     ompi_free_list_init_new( &traffic_msg_ref_free_list,
                              sizeof(ompi_crcp_bkmrk_pml_traffic_message_ref_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_bkmrk_pml_traffic_message_ref_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              32, /* Initial number */
                              -1, /* Max = Unlimited */
                              64, /* Increment by */
@@ -1145,9 +1145,9 @@ int ompi_crcp_bkmrk_pml_init(void) {
     OBJ_CONSTRUCT(&drain_msg_ref_free_list, ompi_free_list_t);
     ompi_free_list_init_new( &drain_msg_ref_free_list,
                              sizeof(ompi_crcp_bkmrk_pml_drain_message_ref_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_bkmrk_pml_drain_message_ref_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              32, /* Initial number */
                              -1, /* Max = Unlimited */
                              64, /* Increment by */
@@ -1156,9 +1156,9 @@ int ompi_crcp_bkmrk_pml_init(void) {
     OBJ_CONSTRUCT(&drain_ack_msg_ref_free_list, ompi_free_list_t);
     ompi_free_list_init_new( &drain_ack_msg_ref_free_list,
                              sizeof(ompi_crcp_bkmrk_pml_drain_message_ack_ref_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_bkmrk_pml_drain_message_ack_ref_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              16, /* Initial number */
                              -1, /* Max = Unlimited */
                              16, /* Increment by */

ompi/mca/mpool/base/mpool_base_tree.c

@@ -80,9 +80,9 @@ int mca_mpool_base_tree_init(void) {
     OBJ_CONSTRUCT(&tree_lock, opal_mutex_t);
     rc = ompi_free_list_init_new(&mca_mpool_base_tree_item_free_list, 
             sizeof(mca_mpool_base_tree_item_t), 
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS(mca_mpool_base_tree_item_t), 
-            0,CACHE_LINE_SIZE,
+            0,opal_cache_line_size,
             0, -1 , 4, NULL);
     if(OMPI_SUCCESS == rc) { 
         rc = ompi_rb_tree_init(&mca_mpool_base_tree, mca_mpool_base_tree_node_compare);

ompi/mca/mpool/rdma/mpool_rdma_module.c

@@ -59,9 +59,9 @@ void mca_mpool_rdma_module_init(mca_mpool_rdma_module_t* mpool)
 
     OBJ_CONSTRUCT(&mpool->reg_list, ompi_free_list_t);
     ompi_free_list_init_new(&mpool->reg_list, mpool->resources.sizeof_reg,
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS(mca_mpool_base_registration_t), 
-            0,CACHE_LINE_SIZE,
+            0,opal_cache_line_size,
             0, -1, 32, NULL);
     OBJ_CONSTRUCT(&mpool->mru_list, opal_list_t);
     OBJ_CONSTRUCT(&mpool->gc_list, opal_list_t);

ompi/mca/mtl/portals/mtl_portals_component.c

@@ -193,9 +193,9 @@ ompi_mtl_portals_component_init(bool enable_progress_threads,
     OBJ_CONSTRUCT(&ompi_mtl_portals.event_fl, ompi_free_list_t);
     ompi_free_list_init_new(&ompi_mtl_portals.event_fl,
                         sizeof(ompi_mtl_portals_event_t),
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(ompi_mtl_portals_event_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         1, -1, 1, NULL);
 
     OBJ_CONSTRUCT(&ompi_mtl_portals.ptl_recv_short_blocks, opal_list_t);

ompi/mca/pml/cm/pml_cm.c

@@ -57,9 +57,9 @@ mca_pml_cm_enable(bool enable)
        maybe? */
     ompi_free_list_init_new(&mca_pml_base_send_requests,
                         sizeof(mca_pml_cm_hvy_send_request_t) + ompi_mtl->mtl_request_size,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_pml_cm_hvy_send_request_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         ompi_pml_cm.free_list_num,
                         ompi_pml_cm.free_list_max,
                         ompi_pml_cm.free_list_inc,
@@ -67,9 +67,9 @@ mca_pml_cm_enable(bool enable)
 
     ompi_free_list_init_new(&mca_pml_base_recv_requests,
                         sizeof(mca_pml_cm_hvy_recv_request_t) + ompi_mtl->mtl_request_size,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_pml_cm_hvy_recv_request_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         ompi_pml_cm.free_list_num,
                         ompi_pml_cm.free_list_max,
                         ompi_pml_cm.free_list_inc,

ompi/mca/pml/crcpw/pml_crcpw_component.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -17,7 +18,7 @@
  */
 
 #include "ompi_config.h"
-#include "opal/sys/cache.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/output.h"
 #include "opal/event/event.h"
 
@@ -142,9 +143,9 @@ mca_pml_base_module_t* mca_pml_crcpw_component_init(int* priority,
         OBJ_CONSTRUCT(&pml_state_list, ompi_free_list_t);
         ompi_free_list_init_new( &pml_state_list,
                              sizeof(ompi_crcp_base_pml_state_t),
-                             CACHE_LINE_SIZE,
+                             opal_cache_line_size,
                              OBJ_CLASS(ompi_crcp_base_pml_state_t),
-                             0,CACHE_LINE_SIZE,
+                             0,opal_cache_line_size,
                              5,  /* Initial number */
                              -1, /* Max = Unlimited */
                              64, /* Increment by */

ompi/mca/pml/csum/pml_csum.c

@@ -97,9 +97,9 @@ int mca_pml_csum_enable(bool enable)
     OBJ_CONSTRUCT(&mca_pml_csum.rdma_frags, ompi_free_list_t);
     ompi_free_list_init_new( &mca_pml_csum.rdma_frags,
                          sizeof(mca_pml_csum_rdma_frag_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_csum_rdma_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_csum.free_list_num,
                          mca_pml_csum.free_list_max,
                          mca_pml_csum.free_list_inc,
@@ -109,9 +109,9 @@ int mca_pml_csum_enable(bool enable)
 
     ompi_free_list_init_new( &mca_pml_csum.recv_frags,
                          sizeof(mca_pml_csum_recv_frag_t) + mca_pml_csum.unexpected_limit,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_csum_recv_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_csum.free_list_num,
                          mca_pml_csum.free_list_max,
                          mca_pml_csum.free_list_inc,
@@ -120,9 +120,9 @@ int mca_pml_csum_enable(bool enable)
     OBJ_CONSTRUCT(&mca_pml_csum.pending_pckts, ompi_free_list_t);
     ompi_free_list_init_new( &mca_pml_csum.pending_pckts,
                          sizeof(mca_pml_csum_pckt_pending_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_csum_pckt_pending_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_csum.free_list_num,
                          mca_pml_csum.free_list_max,
                          mca_pml_csum.free_list_inc,
@@ -134,9 +134,9 @@ int mca_pml_csum_enable(bool enable)
     ompi_free_list_init_new( &mca_pml_csum.send_ranges,
                          sizeof(mca_pml_csum_send_range_t) +
                          (mca_pml_csum.max_send_per_range - 1) * sizeof(mca_pml_csum_com_btl_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_csum_send_range_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_csum.free_list_num,
                          mca_pml_csum.free_list_max,
                          mca_pml_csum.free_list_inc,
@@ -159,9 +159,9 @@ int mca_pml_csum_enable(bool enable)
                          sizeof(mca_pml_csum_send_request_t) +
                          (mca_pml_csum.max_rdma_per_request - 1) *
                          sizeof(mca_pml_csum_com_btl_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_csum_send_request_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_csum.free_list_num,
                          mca_pml_csum.free_list_max,
                          mca_pml_csum.free_list_inc,
@@ -171,9 +171,9 @@ int mca_pml_csum_enable(bool enable)
                          sizeof(mca_pml_csum_recv_request_t) +
                          (mca_pml_csum.max_rdma_per_request - 1) *
                          sizeof(mca_pml_csum_com_btl_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_csum_recv_request_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_csum.free_list_num,
                          mca_pml_csum.free_list_max,
                          mca_pml_csum.free_list_inc,

ompi/mca/pml/csum/pml_csum_component.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2009      IBM Corporation.  All rights reserved.
  * Copyright (c) 2009      Los Alamos National Security, LLC.  All rights
  *                         reserved. 
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -22,7 +22,6 @@
  */
 
 #include "ompi_config.h"
-#include "opal/sys/cache.h"
 #include "opal/event/event.h"
 #include "mpi.h"
 #include "ompi/runtime/params.h"

ompi/mca/pml/dr/pml_dr.c

@@ -76,9 +76,9 @@ int mca_pml_dr_enable(bool enable)
     /* requests */
     ompi_free_list_init_new( &mca_pml_base_send_requests,
                          sizeof(mca_pml_dr_send_request_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_dr_send_request_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_dr.free_list_num,
                          mca_pml_dr.free_list_max,
                          mca_pml_dr.free_list_inc,
@@ -86,9 +86,9 @@ int mca_pml_dr_enable(bool enable)
                                                                                                             
     ompi_free_list_init_new( &mca_pml_base_recv_requests,
                          sizeof(mca_pml_dr_recv_request_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_dr_recv_request_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_dr.free_list_num,
                          mca_pml_dr.free_list_max,
                          mca_pml_dr.free_list_inc,
@@ -98,9 +98,9 @@ int mca_pml_dr_enable(bool enable)
     OBJ_CONSTRUCT(&mca_pml_dr.recv_frags, ompi_free_list_t);
     ompi_free_list_init_new( &mca_pml_dr.recv_frags,
                          sizeof(mca_pml_dr_recv_frag_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_dr_recv_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_dr.free_list_num,
                          mca_pml_dr.free_list_max,
                          mca_pml_dr.free_list_inc,
@@ -109,9 +109,9 @@ int mca_pml_dr_enable(bool enable)
     OBJ_CONSTRUCT(&mca_pml_dr.vfrags, ompi_free_list_t);
     ompi_free_list_init_new( &mca_pml_dr.vfrags,
                          sizeof(mca_pml_dr_vfrag_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_dr_vfrag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_dr.free_list_num,
                          mca_pml_dr.free_list_max,
                          mca_pml_dr.free_list_inc,
@@ -247,9 +247,9 @@ int mca_pml_dr_add_procs(ompi_proc_t** procs, size_t nprocs)
     ompi_free_list_init_new(
                         &mca_pml_dr.buffers,
                         sizeof(mca_pml_dr_buffer_t) + mca_pml_dr.eager_limit,
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_pml_dr_buffer_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         0,
                         mca_pml_dr.free_list_max,
                         mca_pml_dr.free_list_inc,

ompi/mca/pml/dr/pml_dr_component.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -17,7 +18,6 @@
  */
 
 #include "ompi_config.h"
-#include "opal/sys/cache.h"
 #include "opal/event/event.h"
 #include "mpi.h"
 #include "ompi/mca/pml/pml.h"

ompi/mca/pml/ob1/pml_ob1.c

@@ -87,9 +87,9 @@ int mca_pml_ob1_enable(bool enable)
     OBJ_CONSTRUCT(&mca_pml_ob1.rdma_frags, ompi_free_list_t);
     ompi_free_list_init_new( &mca_pml_ob1.rdma_frags,
                          sizeof(mca_pml_ob1_rdma_frag_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_ob1_rdma_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_ob1.free_list_num,
                          mca_pml_ob1.free_list_max,
                          mca_pml_ob1.free_list_inc,
@@ -99,9 +99,9 @@ int mca_pml_ob1_enable(bool enable)
 
     ompi_free_list_init_new( &mca_pml_ob1.recv_frags,
                          sizeof(mca_pml_ob1_recv_frag_t) + mca_pml_ob1.unexpected_limit,
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_ob1_recv_frag_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_ob1.free_list_num,
                          mca_pml_ob1.free_list_max,
                          mca_pml_ob1.free_list_inc,
@@ -110,9 +110,9 @@ int mca_pml_ob1_enable(bool enable)
     OBJ_CONSTRUCT(&mca_pml_ob1.pending_pckts, ompi_free_list_t);
     ompi_free_list_init_new( &mca_pml_ob1.pending_pckts,
                          sizeof(mca_pml_ob1_pckt_pending_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_ob1_pckt_pending_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_ob1.free_list_num,
                          mca_pml_ob1.free_list_max,
                          mca_pml_ob1.free_list_inc,
@@ -124,9 +124,9 @@ int mca_pml_ob1_enable(bool enable)
     ompi_free_list_init_new( &mca_pml_ob1.send_ranges,
                          sizeof(mca_pml_ob1_send_range_t) +
                          (mca_pml_ob1.max_send_per_range - 1) * sizeof(mca_pml_ob1_com_btl_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_ob1_send_range_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_ob1.free_list_num,
                          mca_pml_ob1.free_list_max,
                          mca_pml_ob1.free_list_inc,
@@ -149,9 +149,9 @@ int mca_pml_ob1_enable(bool enable)
                          sizeof(mca_pml_ob1_send_request_t) +
                          (mca_pml_ob1.max_rdma_per_request - 1) *
                          sizeof(mca_pml_ob1_com_btl_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_ob1_send_request_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_ob1.free_list_num,
                          mca_pml_ob1.free_list_max,
                          mca_pml_ob1.free_list_inc,
@@ -161,9 +161,9 @@ int mca_pml_ob1_enable(bool enable)
                          sizeof(mca_pml_ob1_recv_request_t) +
                          (mca_pml_ob1.max_rdma_per_request - 1) *
                          sizeof(mca_pml_ob1_com_btl_t),
-                         CACHE_LINE_SIZE,
+                         opal_cache_line_size,
                          OBJ_CLASS(mca_pml_ob1_recv_request_t),
-                         0,CACHE_LINE_SIZE,
+                         0,opal_cache_line_size,
                          mca_pml_ob1.free_list_num,
                          mca_pml_ob1.free_list_max,
                          mca_pml_ob1.free_list_inc,

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved
  * $COPYRIGHT$
  * 
@@ -20,7 +20,6 @@
  */
 
 #include "ompi_config.h"
-#include "opal/sys/cache.h"
 #include "opal/event/event.h"
 #include "mpi.h"
 #include "ompi/runtime/params.h"

ompi/mca/pml/v/mca/vprotocol/pessimist/vprotocol_pessimist_component.c

@@ -98,9 +98,9 @@ static mca_vprotocol_base_module_t *mca_vprotocol_pessimist_component_init( int*
     OBJ_CONSTRUCT(&mca_vprotocol_pessimist.events_pool, ompi_free_list_t);
     ompi_free_list_init_new(&mca_vprotocol_pessimist.events_pool,
                         sizeof(mca_vprotocol_pessimist_event_t),
-                        CACHE_LINE_SIZE,
+                        opal_cache_line_size,
                         OBJ_CLASS(mca_vprotocol_pessimist_event_t),
-                        0,CACHE_LINE_SIZE,
+                        0,opal_cache_line_size,
                         _free_list_num,
                         _free_list_max,
                         _free_list_inc,

ompi/mca/rcache/rb/rcache_rb_tree.c

@@ -36,9 +36,9 @@ int mca_rcache_rb_tree_init(mca_rcache_rb_module_t* rcache) {
     OBJ_CONSTRUCT(&rcache->rb_tree_item_list, ompi_free_list_t);
     ompi_free_list_init_new(&rcache->rb_tree_item_list, 
             sizeof(mca_rcache_rb_tree_item_t), 
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS(mca_rcache_rb_tree_item_t), 
-            0,CACHE_LINE_SIZE,
+            0,opal_cache_line_size,
             0, -1, 32, NULL); 
     
     return ompi_rb_tree_init(&rcache->rb_tree, 

opal/class/opal_free_list.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -19,8 +20,7 @@
 #include "opal_config.h"
 
 #include "opal/class/opal_free_list.h"
-#include "opal/sys/cache.h"
+#include "opal/runtime/opal.h"
-
 
 static void opal_free_list_construct(opal_free_list_t* fl);
 static void opal_free_list_destruct(opal_free_list_t* fl);
@@ -92,7 +92,7 @@ int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements)
 
     alloc_ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) + 
                                         sizeof(opal_list_item_t) +
-                                        CACHE_LINE_SIZE);
+                                        opal_cache_line_size);
     if(NULL == alloc_ptr)
         return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
 
@@ -102,9 +102,9 @@ int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements)
     opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
     ptr = alloc_ptr + sizeof(opal_list_item_t);
 
-    mod = (uintptr_t)ptr % CACHE_LINE_SIZE;
+    mod = (uintptr_t)ptr % opal_cache_line_size;
     if(mod != 0) {
-        ptr += (CACHE_LINE_SIZE - mod);
+        ptr += (opal_cache_line_size - mod);
     }
 
     if (NULL != flist->fl_elem_class) {

opal/include/opal/sys/Makefile.am

@@ -9,6 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -22,7 +23,6 @@ headers += \
 	opal/sys/architecture.h \
 	opal/sys/atomic.h \
 	opal/sys/atomic_impl.h \
-	opal/sys/cache.h \
 	opal/sys/timer.h
 
 include opal/sys/alpha/Makefile.am

opal/include/opal/sys/cache.h

@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-/** @file
- *
- *  Define the cache characteristics of the platform
- *  
- */
-
-#ifndef OPAL_SYS_CACHE_H
-#define OPAL_SYS_CACHE_H 1
-
-/**
- * Size of cache line.
- *
- * For now hardwire this to a reasonable value, and automate later - RLG
- */
-#define CACHE_LINE_SIZE 128
-
-
-#endif /* OPAL_SYS_CACHE_H */

opal/runtime/opal.h

@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008	   Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2010	   Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -33,6 +34,13 @@ OPAL_DECLSPEC extern const char opal_version_string[];
 OPAL_DECLSPEC extern bool opal_profile;
 OPAL_DECLSPEC extern char *opal_profile_file;
 
+/* Size of a cache line.  To be replaced with real hwloc info (in
+   trunk/v1.5 and beyond, only), but for the moment, just move it here
+   so that we can remove opal/include/sys/cache.h whose only purpose
+   is life is to #define CACHE_LINE_SIZE (and has a conflict on
+   NetBSD). */
+OPAL_DECLSPEC extern int opal_cache_line_size;
+
 /**
  * Initialize the OPAL layer, including the MCA system.
  *

opal/runtime/opal_init.c

@@ -66,6 +66,7 @@ int opal_initialized = 0;
 int opal_util_initialized = 0;
 bool opal_profile = false;
 char *opal_profile_file = NULL;
+int opal_cache_line_size;
 
 static const char *
 opal_err2str(int errnum)
@@ -218,6 +219,12 @@ opal_init_util(int* pargc, char*** pargv)
         return OPAL_SUCCESS;
     }
 
+    /* JMS See note in runtime/opal.h -- this is temporary; to be
+       replaced with real hwloc information soon (in trunk/v1.5 and
+       beyond, only).  This *used* to be a #define, so it's important
+       to define it very early.  */
+    opal_cache_line_size = 128;
+
     /* initialize the memory allocator */
     opal_malloc_init();
 

test/class/ompi_rb_tree.c

@@ -315,9 +315,9 @@ void test2(void)
     
     OBJ_CONSTRUCT(&key_list, ompi_free_list_t);
     ompi_free_list_init_new(&key_list, sizeof(ompi_test_rb_value_t),
-            CACHE_LINE_SIZE,
+            opal_cache_line_size,
             OBJ_CLASS(ompi_test_rb_value_t), 
-            0,CACHE_LINE_SIZE,
+            0,opal_cache_line_size,
             0, -1 , 128, NULL);
     
     OBJ_CONSTRUCT(&tree, ompi_rb_tree_t);