From aba398ce09009c17c17f50f9b23c5af16599aeeb Mon Sep 17 00:00:00 2001
From: Jeff Squyres <jsquyres@cisco.com>
Date: Tue, 24 Apr 2012 17:31:06 +0000
Subject: [PATCH] Per RFC
 (http://www.open-mpi.org/community/lists/devel/2012/04/10905.php), set
 opal_cache_line_size via hwloc data, if we have it. opal_cache_line_size will
 be set to an hwloc-inspired value by the end of orte_init(), but will always
 have a safe value to use (i.e., a default value 128) -- even before
 opal_init() has completed.

Default to the same value of 128 that Open MPI has used for several
years if a) we have no hwloc data, or b) we weren't able to find L2
objects in the hwloc data.

This commit was SVN r26322.
---
 opal/mca/hwloc/base/hwloc_base_util.c | 43 ++++++++++++++++++++++++++-
 opal/runtime/opal.h                   | 12 ++++----
 opal/runtime/opal_init.c              | 11 +++----
 3 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/opal/mca/hwloc/base/hwloc_base_util.c b/opal/mca/hwloc/base/hwloc_base_util.c
index bef3d48f3f..8ca7afbdb8 100644
--- a/opal/mca/hwloc/base/hwloc_base_util.c
+++ b/opal/mca/hwloc/base/hwloc_base_util.c
@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2011      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -27,6 +27,7 @@
 #include <unistd.h>
 #endif
 
+#include "opal/runtime/opal.h"
 #include "opal/constants.h"
 #include "opal/util/argv.h"
 #include "opal/util/output.h"
@@ -169,6 +170,39 @@ int opal_hwloc_base_filter_cpus(hwloc_topology_t topo)
     return OPAL_SUCCESS;
 }
 
+static void fill_cache_line_size(void)
+{
+    int i = 0;
+    unsigned size;
+    hwloc_obj_t obj;
+    bool found = false;
+
+    /* Look for the smallest L2 cache size */
+    size = 4096;
+    while (1) {
+        obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
+                                              HWLOC_OBJ_CACHE, 2,
+                                              i, OPAL_HWLOC_LOGICAL);
+        if (NULL == obj) {
+            break;
+        } else { 
+            found = true;
+            if (NULL != obj->attr &&
+                size > obj->attr->cache.linesize) {
+                size = obj->attr->cache.linesize;
+            }
+        }
+        ++i;
+    }
+
+    /* If we found an L2 cache size in the hwloc data, save it in
+       opal_cache_line_size.  Otherwise, we'll leave whatever default
+       was set in opal_init.c */
+    if (found) {
+        opal_cache_line_size = (int) size;
+    }
+}
+
 int opal_hwloc_base_get_topology(void)
 {
     int rc;
@@ -186,6 +220,13 @@ int opal_hwloc_base_get_topology(void)
 
     /* filter the cpus thru any default cpu set */
     rc = opal_hwloc_base_filter_cpus(opal_hwloc_topology);
+    if (OPAL_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* fill opal_cache_line_size global with the smallest L1 cache
+       line size */
+    fill_cache_line_size();
     return rc;
 }
 
diff --git a/opal/runtime/opal.h b/opal/runtime/opal.h
index 5fa585e029..a3141b70b3 100644
--- a/opal/runtime/opal.h
+++ b/opal/runtime/opal.h
@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008	   Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010	   Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -30,11 +30,11 @@ BEGIN_C_DECLS
 /** version string of opal */
 OPAL_DECLSPEC extern const char opal_version_string[];
 
-/* Size of a cache line.  To be replaced with real hwloc info (in
-   trunk/v1.5 and beyond, only), but for the moment, just move it here
-   so that we can remove opal/include/sys/cache.h whose only purpose
-   is life is to #define CACHE_LINE_SIZE (and has a conflict on
-   NetBSD). */
+/* Size of a cache line.  Initialized to a fixed value (see
+   opal_init.c) until hwloc data is available, at which time it is
+   filled with the smallest size of the lowest cache line (e.g., the
+   smallest line size from all L1 caches found on the current
+   system). */
 OPAL_DECLSPEC extern int opal_cache_line_size;
 
 /**
diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c
index 427b794ba9..8d909f0701 100644
--- a/opal/runtime/opal_init.c
+++ b/opal/runtime/opal_init.c
@@ -70,7 +70,10 @@ const char opal_version_string[] = OPAL_IDENT_STRING;
 
 int opal_initialized = 0;
 int opal_util_initialized = 0;
-int opal_cache_line_size;
+/* We have to put a guess in here in case hwloc is not available.  If
+   hwloc is available, this value will be overwritten when the
+   hwloc data is loaded. */
+int opal_cache_line_size = 128;
 
 static int
 opal_err2str(int errnum, const char **errmsg)
@@ -232,12 +235,6 @@ opal_init_util(int* pargc, char*** pargv)
         return OPAL_SUCCESS;
     }
 
-    /* JMS See note in runtime/opal.h -- this is temporary; to be
-       replaced with real hwloc information soon (in trunk/v1.5 and
-       beyond, only).  This *used* to be a #define, so it's important
-       to define it very early.  */
-    opal_cache_line_size = 128;
-
     /* initialize the memory allocator */
     opal_malloc_init();