Introduce a new debugging MCA parameter:

mpi_show_mpi_alloc_mem_leaks

When activated, MPI_FINALIZE displays a list of memory allocations
from MPI_ALLOC_MEM that were not freed by MPI_FREE_MEM (in each MPI
process).

 * If set to a positive integer, display only that many leaks.
 * If set to a negative integer, display all leaks.
 * If set to 0, do not show any leaks.

This commit was SVN r15736.

ompi/mca/mpool/Makefile.am

@@ -9,6 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -35,6 +36,7 @@ else
 ompidir = $(includedir)
 endif
 
+dist_pkgdata_DATA =
 include base/Makefile.am
 
 distclean-local:

ompi/mca/mpool/base/Makefile.am

@@ -9,6 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -28,3 +29,6 @@ libmca_mpool_la_SOURCES += \
         base/mpool_base_alloc.c \
         base/mpool_base_mem_cb.c \
 	base/mpool_base_tree.c
+
+dist_pkgdata_DATA += \
+        base/help-mpool-base.txt

ompi/mca/mpool/base/help-mpool-base.txt

@@ -0,0 +1,32 @@
+# -*- text -*-
+#
+# Copyright (c) 2007 Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+[all mem leaks]
+The following memory locations were allocated via MPI_ALLOC_MEM but
+not freed via MPI_FREE_MEM before invoking MPI_FINALIZE:
+
+Process ID: %s
+Hostname:   %s
+PID:        %d
+
+%s
+#
+[some mem leaks]
+The following memory locations were allocated via MPI_ALLOC_MEM but
+not freed via MPI_FREE_MEM before invoking MPI_FINALIZE:
+
+Process ID: %s
+Hostname:   %s
+PID:        %d
+
+%s
+
+%d additional leak%s recorded but %s not displayed here.  Set the MCA
+parameter mpi_show_mpi_alloc_mem_leaks to a larger number to see that
+many leaks, or set it to a negative number to see all leaks.

ompi/mca/mpool/base/mpool_base_alloc.c

@@ -124,6 +124,7 @@ void *mca_mpool_base_alloc(size_t size, ompi_info_t *info)
     if(!mpool_tree_item)
         goto out;
 
+    mpool_tree_item->num_bytes = size;
     mpool_tree_item->count = 0;
     
     if(&ompi_mpi_info_null == info)

ompi/mca/mpool/base/mpool_base_tree.c

@@ -1,29 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.5A
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
 /**
-  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-  *                         University Research and Technology
-  *                         Corporation.  All rights reserved.
-  * Copyright (c) 2004-2006 The University of Tennessee and The University
-  *                         of Tennessee Research Foundation.  All rights
-  *                         reserved.
-  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
-  *                         University of Stuttgart.  All rights reserved.
-  * Copyright (c) 2004-2005 The Regents of the University of California.
-  *                         All rights reserved.5A
-  * $COPYRIGHT$
-  * 
-  * Additional copyrights may follow
-  * 
-  * $HEADER$
-  */
-/**
-  * @file
-  * Description of the Registration Cache framework
-  */
+ * @file
+ * Description of the Registration Cache framework
+ */
+
 #include "ompi_config.h"
+
 #include "opal/mca/mca.h"
+#include "opal/util/show_help.h"
+#include "orte/mca/ns/ns_types.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/sys_info.h"
+#include "ompi/runtime/params.h"
 #include "mpool_base_tree.h"
 
 
+static int num_leaks = 0;
+static char *leak_msg = NULL;
+
+static int condition(void *value);
+static void action(void *key, void *value);
+
 OBJ_CLASS_INSTANCE(mca_mpool_base_tree_item_t, ompi_free_list_item_t, NULL, NULL); 
 
 /* 
@@ -115,3 +129,71 @@ void mca_mpool_base_tree_item_put(mca_mpool_base_tree_item_t* item) {
     OMPI_FREE_LIST_RETURN(&mca_mpool_base_tree_item_free_list,
                           &(item->super));
 }
+
+
+/*
+ * Print a show_help kind of message for an items still left in the
+ * tree
+ */
+void mca_mpool_base_tree_print(void)
+{
+    /* If they asked to show 0 leaks, then don't show anything.  */
+    if (0 == ompi_debug_show_mpi_alloc_mem_leaks) {
+        return;
+    }
+
+    num_leaks = 0;
+    ompi_rb_tree_traverse(&mca_mpool_base_tree, condition, action);
+
+    if (num_leaks <= ompi_debug_show_mpi_alloc_mem_leaks ||
+        ompi_debug_show_mpi_alloc_mem_leaks < 0) {
+        opal_show_help("help-mpool-base.txt", "all mem leaks",
+                       true, ORTE_NAME_PRINT(orte_process_info.my_name),
+                       orte_system_info.nodename,
+                       orte_process_info.pid, leak_msg);
+    } else {
+        int i = num_leaks - ompi_debug_show_mpi_alloc_mem_leaks;
+        opal_show_help("help-mpool-base.txt", "some mem leaks",
+                       true, ORTE_NAME_PRINT(orte_process_info.my_name),
+                       orte_system_info.nodename, 
+                       orte_process_info.pid, leak_msg, i,
+                       (i > 1) ? "s were" : " was",
+                       (i > 1) ? "are" : "is");
+    }
+    free(leak_msg);
+    leak_msg = NULL;
+}
+
+
+/* Condition function for rb traversal */
+static int condition(void *value)
+{
+    return 1;
+}
+
+
+/* Action function for rb traversal */
+static void action(void *key, void *value)
+{
+    char *tmp;
+    mca_mpool_base_tree_item_t *item = (mca_mpool_base_tree_item_t *) value;
+
+    if (++num_leaks <= ompi_debug_show_mpi_alloc_mem_leaks ||
+        ompi_debug_show_mpi_alloc_mem_leaks < 0) {
+
+        /* We know that we're supposed to make the first one; check on
+           successive items if we're supposed to catenate more
+           notices. */
+        if (NULL == leak_msg) {
+            asprintf(&leak_msg, "    %lu bytes at address 0x%lx",
+                     (unsigned long) item->num_bytes,
+                     (unsigned long) key);
+        } else {
+            asprintf(&tmp, "%s\n    %lu bytes at address 0x%lx",
+                     leak_msg, (unsigned long) item->num_bytes,
+                     (unsigned long) key);
+            free(leak_msg);
+            leak_msg = tmp;
+        }
+    }
+}

ompi/mca/mpool/base/mpool_base_tree.h

@@ -1,23 +1,27 @@
-/**
-  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-  *                         University Research and Technology
-  *                         Corporation.  All rights reserved.
-  * Copyright (c) 2004-2006 The University of Tennessee and The University
-  *                         of Tennessee Research Foundation.  All rights
-  *                         reserved.
-  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
-  *                         University of Stuttgart.  All rights reserved.
-  * Copyright (c) 2004-2006 The Regents of the University of California.
-  *                         All rights reserved.
-  * $COPYRIGHT$
-  * 
-  * Additional copyrights may follow
-  * 
-  * $HEADER$
-  */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
 
 #ifndef MCA_MPOOL_BASE_TREE_H
 #define MCA_MPOOL_BASE_TREE_H
+
+#include "ompi_config.h"
+
 #define MCA_MPOOL_BASE_TREE_MAX 8
 #include "opal/mca/mca.h"
 #include "ompi/info/info.h"
@@ -25,6 +29,8 @@
 #include "ompi/class/ompi_rb_tree.h"
 #include "ompi/mca/mpool/mpool.h"
 
+BEGIN_C_DECLS
+
 /*
  * Data structures for the tree of allocated memory
  * used for MPI_Alloc_mem and MPI_Free_mem 
@@ -36,10 +42,13 @@
 struct mca_mpool_base_tree_item_t
 {
     ompi_free_list_item_t super;   /**< the parent class */
-    void* key; /* the address this was alloc'd on */
+    void* key; /**< the address this was alloc'd on */
+    size_t num_bytes; /**< the number of bytes in this alloc, only for
+                           debugging reporting with
+                           mpi_show_mpi_alloc_mem_leaks */
     mca_mpool_base_module_t* mpools[MCA_MPOOL_BASE_TREE_MAX]; /**< the mpools */
     mca_mpool_base_registration_t* regs[MCA_MPOOL_BASE_TREE_MAX]; /**< the registrations */
-    uint8_t count;
+    uint8_t count; /**< length of the mpools/regs array */
 };
 typedef struct mca_mpool_base_tree_item_t mca_mpool_base_tree_item_t;
 
@@ -76,6 +85,12 @@ mca_mpool_base_tree_item_t* mca_mpool_base_tree_item_get(void);
  */
 void mca_mpool_base_tree_item_put(mca_mpool_base_tree_item_t* item); 
 
+/*
+ * For debugging, print a show_help kind of message if there are items
+ * left in the tree.
+ */
+void mca_mpool_base_tree_print(void);
 
+END_C_DECLS
 
 #endif /* MCA_MPOOL_BASE_TREE_H */

ompi/mca/mpool/rdma/mpool_rdma_module.c

@@ -423,8 +423,6 @@ void mca_mpool_rdma_finalize(struct mca_mpool_base_module_t *mpool)
 
         if(reg->ref_count) {
             reg->ref_count = 0; /* otherway dereg will fail on assert */
-            opal_output(0, "%s Warning ! Found not released memory\n",
-                    ORTE_NAME_PRINT(orte_process_info.my_name));
         } else if (mca_mpool_rdma_component.leave_pinned) {
             opal_list_remove_item(&mpool_rdma->mru_list,
                     (opal_list_item_t*)reg);

ompi/runtime/ompi_mpi_finalize.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
  *                         reserved. 
  * Copyright (c) 2006      University of Houston. All rights reserved.
@@ -78,6 +78,8 @@
 #include "ompi/mca/mpool/base/base.h"
 #include "ompi/mca/rcache/base/base.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/mpool/base/mpool_base_tree.h"
 
 #if OPAL_ENABLE_FT == 1
 #include "ompi/mca/crcp/crcp.h"
@@ -200,6 +202,12 @@ int ompi_mpi_finalize(void)
 	return ret;
     }
 
+    /* If requested, print out a list of memory allocated by ALLOC_MEM
+       but not freed by FREE_MEM */
+    if (0 != ompi_debug_show_mpi_alloc_mem_leaks) {
+        mca_mpool_base_tree_print();
+    }
+
     /* Now that all MPI objects dealing with communications are gone,
        shut down MCA types having to do with communications */
     if (OMPI_SUCCESS != (ret = mca_pml_base_close())) {

ompi/runtime/ompi_mpi_params.c

@@ -44,6 +44,7 @@
  */
 bool ompi_mpi_param_check = true;
 bool ompi_debug_show_handle_leaks = false;
+int ompi_debug_show_mpi_alloc_mem_leaks = 0;
 bool ompi_debug_no_free_handles = false;
 bool ompi_mpi_show_mca_params = false;
 char *ompi_mpi_show_mca_params_file = NULL;
@@ -121,6 +122,14 @@ int ompi_mpi_register_params(void)
         }
     }
 
+    /* Whether or not to show MPI_ALLOC_MEM leaks */
+
+    mca_base_param_reg_int_name("mpi", "show_mpi_alloc_mem_leaks",
+                                "If >0, MPI_FINALIZE will show up to this many instances of memory allocated by MPI_ALLOC_MEM that was not freed by MPI_FREE_MEM",
+                                false, false, 
+                                ompi_debug_show_mpi_alloc_mem_leaks,
+                                &ompi_debug_show_mpi_alloc_mem_leaks);
+
     /* Whether or not to print all MCA parameters in MPI_INIT */
     mca_base_param_reg_int_name("mpi", "show_mca_params",
                                 "Whether to show all MCA parameter value during MPI_INIT or not (good for reproducability of MPI jobs)",

ompi/runtime/params.h

@@ -56,6 +56,16 @@ OMPI_DECLSPEC extern bool ompi_mpi_param_check;
  */
 OMPI_DECLSPEC extern bool ompi_debug_show_handle_leaks;
 
+/**
+ * If > 0, show that many MPI_ALLOC_MEM leaks during MPI_FINALIZE.  If
+ * enabled, memory that was returned via MPI_ALLOC_MEM but was never
+ * freed via MPI_FREE_MEM will be displayed during MPI_FINALIZE.
+ *
+ * This is good debugging for user applications to find out if they
+ * are inadvertantly orphaning MPI "special" memory.
+ */
+OMPI_DECLSPEC extern int ompi_debug_show_mpi_alloc_mem_leaks;
+
 /**
  * Whether or not to actually free MPI handles when their
  * corresponding destructor is invoked.  If enabled, Open MPI will not