We (Ralph and Jeff) discovered that if the OPAL_DESTDIR environment

variable was set, it was prefixed to ''all'' values in the wrapper
compiler data text files.  For example, if OPAL_DESTDIR was set to
/tmp/bogus and a wrapper compiler data file contained the line:

  preprocessor_flags=-pthread

The value would be exanded to:

  /tmp/bogus/-pthread

Which is clearly wrong.  After some back-and-forth with Ralph and
Brian, Brian submitted this patch that fixes the problem.  Now we
handle three cases properly (assume that configure was invoked with
--prefix=/opt/openmpi and no other directory specifications, and
$OPAL_DESTDIR is set to /tmp/buildroot):

1. Individual directories, such as libdir.  These need to be prepended
with DESTDIR.  I.e., return /tmp/buildroot/opt/openmpi/lib.

2. Compiler flags that have ${FIELD} values embedded in them.  For
example, consider if a wrapper compiler data file contains the
line:

  preprocessor_flags=-DMYFLAG="${prefix}/share/randomthingy/"

The value we should return is:

  -DMYFLAG="/tmp/buildroot/opt/openmpi/share/randomthingy/"

3. Compiler flags that do not have any ${FIELD} values.  For example,
consider if a wrapper compiler data file contains the line:

  preprocessor_flags=-pthread

The value we should return is:

  -pthread

Note, too, that this OPAL_DESTDIR futzing only needs to occur during
opal_init().  By the time opal_init() has completed, all values should
be substituted in that need substituting.  Hence, we take an extra
parameter (is_setup) to know whether we should do this futzing or
not.

This commit was SVN r23402.

opal/mca/installdirs/base/base.h

@@ -1,7 +1,8 @@
 /*
  * Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
  *                         reserved. 
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Sandia National Laboratories. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -25,6 +26,13 @@ BEGIN_C_DECLS
 OPAL_DECLSPEC int opal_installdirs_base_open(void);
 OPAL_DECLSPEC int opal_installdirs_base_close(void);
 
+/* Just like opal_install_dirs_expand() (see installdirs.h), but will
+   also insert the value of the environment variable $OPAL_DESTDIR, if
+   it exists/is set.  This function should *only* be used during the
+   setup routines of installdirs. */
+char * opal_install_dirs_expand_setup(const char* input);
+
+
 /*
  * Globals
  */

opal/mca/installdirs/base/installdirs_base_components.c

@@ -2,6 +2,7 @@
  * Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
  *                         reserved. 
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Sandia National Laboratories. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -91,39 +92,39 @@ opal_installdirs_base_open(void)
 
     /* expand out all the fields */
     opal_install_dirs.prefix = 
-        opal_install_dirs_expand(opal_install_dirs.prefix);
+        opal_install_dirs_expand_setup(opal_install_dirs.prefix);
     opal_install_dirs.exec_prefix = 
-        opal_install_dirs_expand(opal_install_dirs.exec_prefix);
+        opal_install_dirs_expand_setup(opal_install_dirs.exec_prefix);
     opal_install_dirs.bindir = 
-        opal_install_dirs_expand(opal_install_dirs.bindir);
+        opal_install_dirs_expand_setup(opal_install_dirs.bindir);
     opal_install_dirs.sbindir = 
-        opal_install_dirs_expand(opal_install_dirs.sbindir);
+        opal_install_dirs_expand_setup(opal_install_dirs.sbindir);
     opal_install_dirs.libexecdir = 
-        opal_install_dirs_expand(opal_install_dirs.libexecdir);
+        opal_install_dirs_expand_setup(opal_install_dirs.libexecdir);
     opal_install_dirs.datarootdir = 
-        opal_install_dirs_expand(opal_install_dirs.datarootdir);
+        opal_install_dirs_expand_setup(opal_install_dirs.datarootdir);
     opal_install_dirs.datadir = 
-        opal_install_dirs_expand(opal_install_dirs.datadir);
+        opal_install_dirs_expand_setup(opal_install_dirs.datadir);
     opal_install_dirs.sysconfdir = 
-        opal_install_dirs_expand(opal_install_dirs.sysconfdir);
+        opal_install_dirs_expand_setup(opal_install_dirs.sysconfdir);
     opal_install_dirs.sharedstatedir = 
-        opal_install_dirs_expand(opal_install_dirs.sharedstatedir);
+        opal_install_dirs_expand_setup(opal_install_dirs.sharedstatedir);
     opal_install_dirs.localstatedir = 
-        opal_install_dirs_expand(opal_install_dirs.localstatedir);
+        opal_install_dirs_expand_setup(opal_install_dirs.localstatedir);
     opal_install_dirs.libdir = 
-        opal_install_dirs_expand(opal_install_dirs.libdir);
+        opal_install_dirs_expand_setup(opal_install_dirs.libdir);
     opal_install_dirs.includedir = 
-        opal_install_dirs_expand(opal_install_dirs.includedir);
+        opal_install_dirs_expand_setup(opal_install_dirs.includedir);
     opal_install_dirs.infodir = 
-        opal_install_dirs_expand(opal_install_dirs.infodir);
+        opal_install_dirs_expand_setup(opal_install_dirs.infodir);
     opal_install_dirs.mandir = 
-        opal_install_dirs_expand(opal_install_dirs.mandir);
+        opal_install_dirs_expand_setup(opal_install_dirs.mandir);
     opal_install_dirs.pkgdatadir = 
-        opal_install_dirs_expand(opal_install_dirs.pkgdatadir);
+        opal_install_dirs_expand_setup(opal_install_dirs.pkgdatadir);
     opal_install_dirs.pkglibdir = 
-        opal_install_dirs_expand(opal_install_dirs.pkglibdir);
+        opal_install_dirs_expand_setup(opal_install_dirs.pkglibdir);
     opal_install_dirs.pkgincludedir = 
-        opal_install_dirs_expand(opal_install_dirs.pkgincludedir);
+        opal_install_dirs_expand_setup(opal_install_dirs.pkgincludedir);
 
 #if 0
     fprintf(stderr, "prefix:         %s\n", opal_install_dirs.prefix);

opal/mca/installdirs/base/installdirs_base_expand.c

@@ -1,8 +1,9 @@
 /*
  * Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
  *                         reserved. 
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007      Sun Microsystem, Inc.  All rights reserved.
+ * Copyright (c) 2010      Sandia National Laboratories. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -34,19 +35,65 @@
     } while (0)
 
 
-
-char *
-opal_install_dirs_expand(const char* input)
+/*
+ * Read the lengthy comment below to understand the value of the
+ * is_setup parameter.
+ */
+static char *
+opal_install_dirs_expand_internal(const char* input, bool is_setup)
 {
     size_t len, i;
     bool needs_expand = false;
     char *retval = NULL;
-    char *destdir = getenv("OPAL_DESTDIR");
+    char *destdir = NULL;
     size_t destdir_offset = 0;
 
+    /* This is subtle, and worth explaining.  
+
+       If we substitute in any ${FIELD} values, we need to prepend it
+       with the value of the $OPAL_DESTDIR environment variable -- if
+       it is set.  
+
+       We need to handle at least three cases properly (assume that
+       configure was invoked with --prefix=/opt/openmpi and no other
+       directory specifications, and OPAL_DESTDIR is set to
+       /tmp/buildroot):
+
+       1. Individual directories, such as libdir.  These need to be
+          prepended with DESTDIR.  I.e., return
+          /tmp/buildroot/opt/openmpi/lib.
+
+       2. Compiler flags that have ${FIELD} values embedded in them.
+          For example, consider if a wrapper compiler data file
+          contains the line:
+
+          preprocessor_flags=-DMYFLAG="${prefix}/share/randomthingy/"
+
+          The value we should return is:
+
+          -DMYFLAG="/tmp/buildroot/opt/openmpi/share/randomthingy/"
+
+       3. Compiler flags that do not have any ${FIELD} values.
+          For example, consider if a wrapper compiler data file
+          contains the line:
+
+          preprocessor_flags=-pthread
+
+          The value we should return is:
+
+          -pthread
+
+       Note, too, that this OPAL_DESTDIR futzing only needs to occur
+       during opal_init().  By the time opal_init() has completed, all
+       values should be substituted in that need substituting.  Hence,
+       we take an extra parameter (is_setup) to know whether we should
+       do this futzing or not. */
+    if (is_setup) {
+        destdir = getenv("OPAL_DESTDIR");
         if (NULL != destdir && strlen(destdir) > 0) {
             destdir_offset = strlen(destdir);
         }
+    }
 
     len = strlen(input);
     for (i = 0 ; i < len ; ++i) {
@@ -93,3 +140,19 @@ opal_install_dirs_expand(const char* input)
 
     return retval;
 }
+
+
+char *
+opal_install_dirs_expand(const char* input)
+{
+    /* We do NOT want OPAL_DESTDIR expansion in this case. */
+    return opal_install_dirs_expand_internal(input, false);
+}
+
+
+char *
+opal_install_dirs_expand_setup(const char* input)
+{
+    /* We DO want OPAL_DESTDIR expansion in this case. */
+    return opal_install_dirs_expand_internal(input, true);
+}