diff --git a/src/class/ompi_object.h b/src/class/ompi_object.h
index fe0e31cd3d..bd178e3156 100644
--- a/src/class/ompi_object.h
+++ b/src/class/ompi_object.h
@@ -107,7 +107,7 @@
 #include "ompi_config.h"
 #endif
 
-#include "include/atomic.h"
+#include "include/sys/atomic.h"
 
 /*
  * BEGIN_C_DECLS should be used at the beginning of your declarations,
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index 7bd44f8adf..5fa2783560 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -7,7 +7,6 @@ include $(top_srcdir)/config/Makefile.options
 SUBDIRS = sys
 
 noinst_HEADERS = \
-        atomic.h \
         constants.h \
         ompi.h \
         totalview.h \
diff --git a/src/include/atomic.h b/src/include/atomic.h
deleted file mode 100644
index 0f61d7c0c6..0000000000
--- a/src/include/atomic.h
+++ /dev/null
@@ -1,437 +0,0 @@
-/*
- * $HEADER$
- */
-
-/** @file
- *
- * Atomic operations.
- *
- * This API is patterned after the FreeBSD kernel atomic interface
- * (which is influenced by Intel's ia64 architecture).  The
- * FreeBSD interface is documented at
- *
- * http://www.freebsd.org/cgi/man.cgi?query=atomic&sektion=9
- *
- * Only the necessary subset of functions are implemented here.
- */
-
-#ifndef OMPI_ATOMIC_H
-#define OMPI_ATOMIC_H 1
-
-#include "ompi_config.h"
-
-#if 1
-
-/*
- * prototypes
- */
-
-/**
- * Atomic compare and set of unsigned 32-bit integer.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * Pseudo-code:
- *
- * @code
- *   int ompi_atomic_cmpset_32(addr, oldval, newval)
- *   {
- *       if (*addr == oldval) {
- *           *addr = newval;
- *           return 1;  // success, set value
- *       } else {
- *           return 0;  // failure, do not set value
- *       }
- *   }
- * @endcode
- */
-static inline int ompi_atomic_cmpset_32(volatile uint32_t *addr,
-                                        uint32_t oldval,
-                                        uint32_t newval);
-
-
-/**
- * Atomic compare and set of unsigned 32-bit integer with acquire
- * semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_acq_32(volatile uint32_t *addr,
-                                            uint32_t oldval,
-                                            uint32_t newval);
-
-
-/**
- * Atomic compare and set of unsigned 32-bit integer with release
- * semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_rel_32(volatile uint32_t *addr,
-                                            uint32_t oldval,
-                                            uint32_t newval);
-
-
-/**
- * Atomic compare and set of unsigned 64-bit integer.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_64(volatile uint64_t *addr,
-                                        uint64_t oldval,
-                                        uint64_t newval);
-
-
-/**
- * Atomic compare and set of unsigned 64-bit integer with acquire
- * semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_acq_64(volatile uint64_t *addr,
-                                            uint64_t oldval,
-                                            uint64_t newval);
-
-
-/**
- * Atomic compare and set of unsigned 64-bit integer with release
- * semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_rel_64(volatile uint64_t *addr,
-                                            uint64_t oldval,
-                                            uint64_t newval);
-
-
-/**
- * Atomic compare and set of integer.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_int(volatile int *addr,
-                                         int oldval,
-                                         int newval);
-
-
-/**
- * Atomic compare and set of integer with acquire semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_acq_int(volatile int *addr,
-                                             int oldval,
-                                             int newval);
-
-
-/**
- * Atomic compare and set of integer with release semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_rel_int(volatile int *addr,
-                                             int oldval,
-                                             int newval);
-
-
-/**
- * Atomic compare and set of pointer.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_ptr(volatile void *addr,
-                                         void *oldval,
-                                         void *newval);
-
-
-/**
- * Atomic compare and set of pointer with acquire semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_acq_ptr(volatile void *addr,
-                                             void *oldval,
-                                             void *newval);
-
-
-/**
- * Atomic compare and set of pointer with release semantics.
- *
- * @param addr          Address of integer.
- * @param oldval        Comparison value.
- * @param newval        New value to set if comparision is true.
- *
- * See ompi_atomic_cmpset_32 for pseudo-code.
- */
-static inline int ompi_atomic_cmpset_rel_ptr(volatile void *addr,
-                                             void *oldval,
-                                             void *newval);
-
-/**
- * Atomically add to a 32-bit integer.
- *
- * @param addr          Address of integer.
- * @param delta         Value to add.
- * @return              New value of integer.
- */
-static inline uint32_t ompi_atomic_add_32(uint32_t *addr, int delta);
-
-
-/**
- * Atomically add to a 64-bit integer.
- *
- * @param addr          Address of integer.
- * @param delta         Value to add.
- * @return              New value of integer.
- */
-static inline uint64_t ompi_atomic_add_64(uint64_t *addr, int delta);
-
-
-/**
- * Atomically add to an integer.
- *
- * @param addr          Address of integer.
- * @param delta         Value to add.
- * @return              New value of integer.
- */
-static inline int ompi_atomic_add_int(int *addr, int delta);
- 
-
-/*
- * implementation (system specific)
- */
-
-#if defined (__GNUC__)
-
-#if   defined(__alpha__)
-# include "sys/alpha/atomic.h"
-# define OMPI_ATOMIC_OPS 1
-#elif defined(__amd64__)
-# include "sys/amd64/atomic.h"
-# define OMPI_ATOMIC_OPS 1
-#elif defined(__i386__)
-# define OMPI_ATOMIC_OPS 1
-# include "sys/ia32/atomic.h"
-# define OMPI_ATOMIC_OPS 1
-#elif defined(__ia64__)
-# include "sys/ia64/atomic.h"
-# define OMPI_ATOMIC_OPS 1
-#elif defined(__POWERPC__)
-# include "sys/powerpc/atomic.h"
-# define OMPI_ATOMIC_OPS 1
-#elif defined(__sparc64__)
-# include "sys/sparc/atomic.h"
-# define OMPI_ATOMIC_OPS 1
-#endif
-
-#endif
-
-
-/*
- * implementation (derived)
- */
-
-#if SIZEOF_INT == 4
-
-static inline int ompi_atomic_cmpset_int(volatile int *addr,
-                                         int oldval,
-                                         int newval)
-{
-    return ompi_atomic_cmpset_32((volatile uint32_t *) addr,
-                                 (uint32_t) oldval,
-                                 (uint32_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_acq_int(volatile int *addr,
-                                             int oldval,
-                                             int newval)
-{
-    return ompi_atomic_cmpset_acq_32((volatile uint32_t *) addr,
-                                     (uint32_t) oldval,
-                                     (uint32_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_rel_int(volatile int *addr,
-                                             int oldval,
-                                             int newval)
-{
-    return ompi_atomic_cmpset_rel_32((volatile uint32_t *) addr,
-                                     (uint32_t) oldval,
-                                     (uint32_t) newval);
-}
-
-#elif SIZEOF_INT == 8
-
-static inline int ompi_atomic_cmpset_int(volatile int *addr,
-                                         int oldval,
-                                         int newval)
-{
-    return ompi_atomic_cmpset_64((volatile uint64_t *) addr,
-                                 (uint64_t) oldval,
-                                 (uint64_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_acq_int(volatile int *addr,
-                                             int oldval,
-                                             int newval)
-{
-    return ompi_atomic_cmpset_acq_64((volatile uint64_t *) addr,
-                                     (uint64_t) oldval,
-                                     (uint64_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_rel_int(volatile int *addr,
-                                             int oldval,
-                                             int newval)
-{
-    return ompi_atomic_cmpset_rel_64((volatile uint64_t *) addr,
-                                     (uint64_t) oldval,
-                                     (uint64_t) newval);
-}
-
-#else 
-
-#error
-
-#endif
-
-
-#if SIZEOF_VOID_P == 4
-
-static inline int ompi_atomic_cmpset_ptr(volatile void *addr,
-                                         void *oldval,
-                                         void *newval)
-{
-    return ompi_atomic_cmpset_32((volatile uint32_t *) addr,
-                                 (uint32_t) oldval, (uint32_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_acq_ptr(volatile void *addr,
-                                             void *oldval,
-                                             void *newval)
-{
-    return ompi_atomic_cmpset_acq_32((volatile uint32_t *) addr,
-                                     (uint32_t) oldval, (uint32_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_rel_ptr(volatile void *addr,
-                                             void *oldval,
-                                             void *newval)
-{
-    return ompi_atomic_cmpset_rel_32((volatile uint32_t *) addr,
-                                     (uint32_t) oldval, (uint32_t) newval);
-}
-
-#elif SIZEOF_VOID_P == 8
-
-static inline int ompi_atomic_cmpset_ptr(volatile void *addr,
-                                         void *oldval,
-                                         void *newval)
-{
-    return ompi_atomic_cmpset_64((volatile uint64_t *) addr,
-                                 (uint64_t) oldval,
-                                 (uint64_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_acq_ptr(volatile void *addr,
-                                             void *oldval,
-                                             void *newval)
-{
-    return ompi_atomic_cmpset_acq_64((volatile uint64_t *) addr,
-                                     (uint64_t) oldval,
-                                     (uint64_t) newval);
-}
-
-static inline int ompi_atomic_cmpset_rel_ptr(volatile void *addr,
-                                             void *oldval,
-                                             void *newval)
-{
-    return ompi_atomic_cmpset_rel_64((volatile uint64_t *) addr,
-                                     (uint64_t) oldval,
-                                     (uint64_t) newval);
-}
-
-#else
-
-#error
-
-#endif
-
-
-static inline uint32_t ompi_atomic_add_32(uint32_t *addr, int delta)
-{
-    uint32_t oldval;
-
-    do {
-        oldval = *addr;
-    } while (0 == ompi_atomic_cmpset_32(addr, oldval, oldval + delta));
-    return (oldval + delta);
-}
-
-
-static inline uint64_t ompi_atomic_add_64(uint64_t *addr, int delta)
-{
-    uint64_t oldval;
-
-    do {
-        oldval = *addr;
-    } while (0 == ompi_atomic_cmpset_64(addr, oldval, oldval + delta));
-    return (oldval + delta);
-}
-
-
-static inline int ompi_atomic_add_int(int *addr, int delta)
-{
-    int oldval;
-
-    do {
-        oldval = *addr;
-    } while (0 == ompi_atomic_cmpset_int(addr, oldval, oldval + delta));
-    return (oldval + delta);
-}
-
-#endif
-
-#endif /* OMPI_ATOMIC_H */
diff --git a/src/include/sys/Makefile.am b/src/include/sys/Makefile.am
index e122c265c1..1e1f1b3214 100644
--- a/src/include/sys/Makefile.am
+++ b/src/include/sys/Makefile.am
@@ -5,3 +5,14 @@
 include $(top_srcdir)/config/Makefile.options
 
 SUBDIRS = alpha amd64 ia32 ia64 powerpc sparc64
+
+noinst_HEADERS = atomic.h
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/include/sys
+ompi_HEADERS = $(noinst_HEADERS)
+else
+ompidir = $(includedir)
+endif
diff --git a/src/include/sys/alpha/Makefile.am b/src/include/sys/alpha/Makefile.am
index 21afe2b2da..df06adbf27 100644
--- a/src/include/sys/alpha/Makefile.am
+++ b/src/include/sys/alpha/Makefile.am
@@ -4,13 +4,13 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-headers = atomic.h
+noinst_HEADERS = atomic.h
 
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 ompidir = $(includedir)/openmpi/include/sys/alpha
-ompi_HEADERS = $(headers)
+ompi_HEADERS = $(noinst_HEADERS)
 else
 ompidir = $(includedir)
 endif
diff --git a/src/include/sys/alpha/atomic.h b/src/include/sys/alpha/atomic.h
index 9eb6ce73fa..44ab1db9b5 100644
--- a/src/include/sys/alpha/atomic.h
+++ b/src/include/sys/alpha/atomic.h
@@ -2,8 +2,8 @@
  * $HEADER$
  */
 
-#ifndef OMPI_SYS_ATOMIC_H_INCLUDED
-#define OMPI_SYS_ATOMIC_H_INCLUDED
+#ifndef OMPI_SYS_ARCH_ATOMIC_H
+#define OMPI_SYS_ARCH_ATOMIC_H 1
 
 /*
  * On alpha, everything is load-locked, store-conditional...
@@ -134,4 +134,4 @@ static inline int ompi_atomic_cmpset_rel_64(volatile uint64_t *addr,
 }
 
 
-#endif /* ! OMPI_SYS_ATOMIC_H_INCLUDED */
+#endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
diff --git a/src/include/sys/amd64/Makefile.am b/src/include/sys/amd64/Makefile.am
index 8eece66467..014d60e81e 100644
--- a/src/include/sys/amd64/Makefile.am
+++ b/src/include/sys/amd64/Makefile.am
@@ -4,13 +4,13 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-headers = atomic.h
+noinst_HEADERS = atomic.h
 
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 ompidir = $(includedir)/openmpi/include/sys/amd64
-ompi_HEADERS = $(headers)
+ompi_HEADERS = $(noinst_HEADERS)
 else
 ompidir = $(includedir)
 endif
diff --git a/src/include/sys/amd64/atomic.h b/src/include/sys/amd64/atomic.h
index 1b85ff0669..f1b1863215 100644
--- a/src/include/sys/amd64/atomic.h
+++ b/src/include/sys/amd64/atomic.h
@@ -2,8 +2,8 @@
  * $HEADER$
  */
 
-#ifndef OMPI_SYS_ATOMIC_H_INCLUDED
-#define OMPI_SYS_ATOMIC_H_INCLUDED
+#ifndef OMPI_SYS_ARCH_ATOMIC_H
+#define OMPI_SYS_ARCH_ATOMIC_H 1
 
 /*
  * On amd64, we use cmpxchg.
@@ -103,4 +103,4 @@ static inline int ompi_atomic_cmpset_rel_64(volatile uint64_t *addr,
     return ompi_atomic_cmpset_64( addr, oldval, newval );
 }
 
-#endif /* ! OMPI_SYS_ATOMIC_H_INCLUDED */
+#endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
diff --git a/src/include/sys/atomic.h b/src/include/sys/atomic.h
index 044dfee1a5..454842ecd0 100644
--- a/src/include/sys/atomic.h
+++ b/src/include/sys/atomic.h
@@ -15,8 +15,8 @@
  * Only the necessary subset of functions are implemented here.
  */
 
-#ifndef OMPI_ATOMIC_H
-#define OMPI_ATOMIC_H 1
+#ifndef OMPI_SYS_ATOMIC_H
+#define OMPI_SYS_ATOMIC_H 1
 
 #include "ompi_config.h"
 
@@ -251,17 +251,17 @@ STATIC_INLINE int ompi_atomic_add_int(int *addr, int delta);
  */
 
 #if   defined(__alpha__)
-# include "sys/alpha/atomic.h"
+# include "alpha/atomic.h"
 #elif defined(__amd64__)
-# include "sys/amd64/atomic.h"
+# include "amd64/atomic.h"
 #elif defined(__i386__)
-# include "sys/ia32/atomic.h"
+# include "ia32/atomic.h"
 #elif defined(__ia64__)
-# include "sys/ia64/atomic.h"
+# include "ia64/atomic.h"
 #elif defined(__POWERPC__)
-# include "sys/powerpc/atomic.h"
+# include "powerpc/atomic.h"
 #elif defined(__sparc__) || defined(__sparc)
-# include "sys/sparc64/atomic.h"
+# include "sparc64/atomic.h"
 #endif
 
 #endif
@@ -430,4 +430,4 @@ static inline int ompi_atomic_add_int(int *addr, int delta)
     return (oldval + delta);
 }
 
-#endif /* OMPI_ATOMIC_H */
+#endif /* OMPI_SYS_ATOMIC_H */
diff --git a/src/include/sys/cache.h b/src/include/sys/cache.h
index c141fa41dd..c8a802195c 100644
--- a/src/include/sys/cache.h
+++ b/src/include/sys/cache.h
@@ -4,16 +4,19 @@
 
 /** @file
  *
- *  Define the cache caracteristics of the platform
+ *  Define the cache characteristics of the platform
  *  
  */
 
-#ifndef CACHE
-#define CACHE 1
+#ifndef OMPI_SYS_CACHE_H
+#define OMPI_SYS_CACHE_H 1
 
-/* size of cache line  - for now hardwire this to a reasonable
- * value, and automate later - RLG */
+/**
+ * Size of cache line.
+ *
+ * For now hardwire this to a reasonable value, and automate later - RLG
+ */
 #define CACHE_LINE_SIZE 128
 
 
-#endif /* CACHE */
+#endif /* OMPI_SYS_CACHE_H */
diff --git a/src/include/sys/ia32/Makefile.am b/src/include/sys/ia32/Makefile.am
index 507d040b13..d68a0ee2ff 100644
--- a/src/include/sys/ia32/Makefile.am
+++ b/src/include/sys/ia32/Makefile.am
@@ -4,13 +4,13 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-headers = atomic.h
+noinst_HEADERS = atomic.h atomic.s
 
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 ompidir = $(includedir)/openmpi/include/sys/ia32
-ompi_HEADERS = $(headers)
+ompi_HEADERS = $(noinst_HEADERS)
 else
 ompidir = $(includedir)
 endif
diff --git a/src/include/sys/ia32/atomic.h b/src/include/sys/ia32/atomic.h
index f6aa91e1b9..9049f7ff51 100644
--- a/src/include/sys/ia32/atomic.h
+++ b/src/include/sys/ia32/atomic.h
@@ -2,8 +2,8 @@
  * $HEADER$
  */
 
-#ifndef OMPI_SYS_ATOMIC_H_INCLUDED
-#define OMPI_SYS_ATOMIC_H_INCLUDED
+#ifndef OMPI_SYS_ARCH_ATOMIC_H
+#define OMPI_SYS_ARCH_ATOMIC_H 1
 
 /*
  * On ia32, we use cmpxchg.
@@ -109,4 +109,4 @@ static inline int ompi_atomic_cmpset_rel_64(volatile uint64_t *addr,
     return ompi_atomic_cmpset_64(addr, oldval, newval);
 }
 
-#endif /* ! OMPI_SYS_ATOMIC_H_INCLUDED */
+#endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
diff --git a/src/include/sys/ia64/Makefile.am b/src/include/sys/ia64/Makefile.am
index b74bbc7b73..36b989c220 100644
--- a/src/include/sys/ia64/Makefile.am
+++ b/src/include/sys/ia64/Makefile.am
@@ -4,13 +4,13 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-headers = atomic.h
+noinst_HEADERS = atomic.h
 
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 ompidir = $(includedir)/openmpi/include/sys/ia64
-ompi_HEADERS = $(headers)
+ompi_HEADERS = $(noinst_HEADERS)
 else
 ompidir = $(includedir)
 endif
diff --git a/src/include/sys/ia64/atomic.h b/src/include/sys/ia64/atomic.h
index 41704e12a0..2d310d4909 100644
--- a/src/include/sys/ia64/atomic.h
+++ b/src/include/sys/ia64/atomic.h
@@ -2,8 +2,8 @@
  * $HEADER$
  */
 
-#ifndef OMPI_SYS_ATOMIC_H_INCLUDED
-#define OMPI_SYS_ATOMIC_H_INCLUDED
+#ifndef OMPI_SYS_ARCH_ATOMIC_H
+#define OMPI_SYS_ARCH_ATOMIC_H 1
 
 /*
  * On ia64, we use cmpxchg, which supports acquire/release semantics natively.
@@ -118,4 +118,4 @@ static inline int ompi_atomic_cmpset_64(volatile uint64_t *addr,
 }
 
 
-#endif /* ! OMPI_SYS_ATOMIC_H_INCLUDED */
+#endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
diff --git a/src/include/sys/powerpc/Makefile.am b/src/include/sys/powerpc/Makefile.am
index ae63d04628..7c50a11db9 100644
--- a/src/include/sys/powerpc/Makefile.am
+++ b/src/include/sys/powerpc/Makefile.am
@@ -4,13 +4,13 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-headers = atomic.h
+noinst_HEADERS = atomic.h atomic.s
 
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 ompidir = $(includedir)/openmpi/include/sys/powerpc
-ompi_HEADERS = $(headers)
+ompi_HEADERS = $(noinst_HEADERS)
 else
 ompidir = $(includedir)
 endif
diff --git a/src/include/sys/powerpc/atomic.h b/src/include/sys/powerpc/atomic.h
index 754e57332e..efb4900bc8 100644
--- a/src/include/sys/powerpc/atomic.h
+++ b/src/include/sys/powerpc/atomic.h
@@ -2,8 +2,8 @@
  * $HEADER$
  */
 
-#ifndef OMPI_SYS_ATOMIC_H_INCLUDED
-#define OMPI_SYS_ATOMIC_H_INCLUDED
+#ifndef OMPI_SYS_ARCH_ATOMIC_H
+#define OMPI_SYS_ARCH_ATOMIC_H 1
 
 /*
  * On powerpc ...
@@ -128,4 +128,4 @@ static inline int ompi_atomic_cmpset_rel_64(volatile uint64_t *addr,
 }
 
 
-#endif /* ! OMPI_SYS_ATOMIC_H_INCLUDED */
+#endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
diff --git a/src/include/sys/sparc64/Makefile.am b/src/include/sys/sparc64/Makefile.am
index 9768cba290..326463d166 100644
--- a/src/include/sys/sparc64/Makefile.am
+++ b/src/include/sys/sparc64/Makefile.am
@@ -4,13 +4,13 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-headers = atomic.h
+noinst_HEADERS = atomic.h atomic.s
 
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 ompidir = $(includedir)/openmpi/include/sys/sparc64
-ompi_HEADERS = $(headers)
+ompi_HEADERS = $(noinst_HEADERS)
 else
 ompidir = $(includedir)
 endif
diff --git a/src/include/sys/sparc64/atomic.h b/src/include/sys/sparc64/atomic.h
index 3b6c5df08c..cf60c4db5e 100644
--- a/src/include/sys/sparc64/atomic.h
+++ b/src/include/sys/sparc64/atomic.h
@@ -2,8 +2,8 @@
  * $HEADER$
  */
 
-#ifndef OMPI_SYS_ATOMIC_H_INCLUDED
-#define OMPI_SYS_ATOMIC_H_INCLUDED
+#ifndef OMPI_SYS_ARCH_ATOMIC_H
+#define OMPI_SYS_ARCH_ATOMIC_H 1
 
 /*
  * On sparc64, use casa and casxa (compare and swap) instructions.
@@ -106,4 +106,4 @@ static inline int ompi_atomic_cmpset_rel_64(volatile uint64_t *addr,
 }
 
 
-#endif /* ! OMPI_SYS_ATOMIC_H_INCLUDED */
+#endif /* ! OMPI_SYS_ARCH_ATOMIC_H */
diff --git a/src/mca/pml/teg/src/pml_teg_proc.c b/src/mca/pml/teg/src/pml_teg_proc.c
index ab0a231dd7..12689f7ea6 100644
--- a/src/mca/pml/teg/src/pml_teg_proc.c
+++ b/src/mca/pml/teg/src/pml_teg_proc.c
@@ -2,7 +2,9 @@
  * $HEADER$
  */
 
-#include "include/atomic.h"
+#include "ompi_config.h"
+
+#include "include/sys/atomic.h"
 #include "pml_teg.h"
 #include "pml_teg_proc.h"
 #include "pml_ptl_array.h"
diff --git a/src/mca/ptl/elan/src/ptl_elan_proc.c b/src/mca/ptl/elan/src/ptl_elan_proc.c
index 9f3f658802..a319822e6a 100644
--- a/src/mca/ptl/elan/src/ptl_elan_proc.c
+++ b/src/mca/ptl/elan/src/ptl_elan_proc.c
@@ -1,8 +1,12 @@
 /*
  * $HEADER$
  */
+
+#include "ompi_config.h"
+
 #include <string.h>
-#include "atomic.h"
+
+#include "sys/atomic.h"
 #include "class/ompi_hash_table.h"
 #include "mca/base/mca_base_module_exchange.h"
 #include "ptl_elan.h"
diff --git a/src/mca/ptl/gm/src/ptl_gm_proc.c b/src/mca/ptl/gm/src/ptl_gm_proc.c
index a4ff2eb25c..57c439cf6f 100644
--- a/src/mca/ptl/gm/src/ptl_gm_proc.c
+++ b/src/mca/ptl/gm/src/ptl_gm_proc.c
@@ -1,8 +1,12 @@
 /*
  * $HEADER$
  */
+
+#include "ompi_config.h"
+
 #include <string.h>
-#include "atomic.h"
+
+#include "sys/atomic.h"
 #include "class/ompi_hash_table.h"
 #include "mca/base/mca_base_module_exchange.h"
 #include "ptl_gm.h"
diff --git a/src/mca/ptl/ib/src/ptl_ib_proc.c b/src/mca/ptl/ib/src/ptl_ib_proc.c
index be4c103538..02ea402dda 100644
--- a/src/mca/ptl/ib/src/ptl_ib_proc.c
+++ b/src/mca/ptl/ib/src/ptl_ib_proc.c
@@ -1,4 +1,6 @@
-#include "include/atomic.h"
+#include "ompi_config.h"
+
+#include "include/sys/atomic.h"
 #include "class/ompi_hash_table.h"
 #include "mca/base/mca_base_module_exchange.h"
 
diff --git a/src/mca/ptl/tcp/src/ptl_tcp_proc.c b/src/mca/ptl/tcp/src/ptl_tcp_proc.c
index a2d61f8089..143802facb 100644
--- a/src/mca/ptl/tcp/src/ptl_tcp_proc.c
+++ b/src/mca/ptl/tcp/src/ptl_tcp_proc.c
@@ -1,8 +1,12 @@
 /*
  * $HEADER$
  */
+
+#include "ompi_config.h"
+
 #include <string.h>
-#include "include/atomic.h"
+
+#include "include/sys/atomic.h"
 #include "class/ompi_hash_table.h"
 #include "mca/base/mca_base_module_exchange.h"
 #include "ptl_tcp.h"
diff --git a/src/util/Makefile.am b/src/util/Makefile.am
index db5a04e307..4abda1dab3 100644
--- a/src/util/Makefile.am
+++ b/src/util/Makefile.am
@@ -31,6 +31,7 @@ headers = \
 
 libutil_la_SOURCES = \
     $(headers) \
+    asm.S \
     argv.c \
     cmd_line.c \
     common_cmd_line.c \
diff --git a/test/include/atomic.c b/test/include/atomic.c
index f96f2855e7..0f25cea210 100644
--- a/test/include/atomic.c
+++ b/test/include/atomic.c
@@ -7,7 +7,7 @@
 
 #include "support.h"
 
-#include "include/atomic.h"
+#include "include/sys/atomic.h"
 
 /* default options */