Fixes trac:580

* Add MPI::Status methods Set_elements() and Set_cancelled()
 * Added a bunch of comments in various places in the MPI C++ bindings
   implementatio just to explain what's going on (because C++ can hide
   a lot from you)
 * Insert C++ callbacks for the MPI_Grequest callback functions
   registered by MPI::Grequest::Start().  These callbacks keep a
   little meta-data (created by Grequest::Start()) that allow the
   proper callback signatures from C (i.e., from ompi_grequest_<foo>()
   in libmpi.a -- C code), translate arguments as required, and then
   invoke the callbacks with proper C++ signatures (i.e., call
   user-defined callbacks with C++ function signatures).

This commit was SVN r12446.

The following Trac tickets were found above:
  Ticket 580 --> https://svn.open-mpi.org/trac/ompi/ticket/580

ompi/mpi/cxx/intercepts.cc

@@ -10,6 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
+// Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -285,3 +286,37 @@ ompi_mpi_cxx_delete_attr_intercept(MPI_Comm comm, int keyval,
   return ret; 
 }
 
+// For similar reasons as above, we need to intercept calls for the 3
+// generalized request callbacks (convert arguments to C++ types and
+// invoke the C++ callback signature).
+
+extern "C" int
+ompi_mpi_cxx_grequest_query_fn_intercept(void *state, MPI_Status *status)
+{
+    struct MPI::Grequest_intercept_t *data = 
+        (struct MPI::Grequest_intercept_t *) state;
+
+    MPI::Status s(*status);
+    int ret = data->git_cxx_query_fn(data->git_extra, s);
+    *status = s;
+    return ret;
+}
+
+extern "C" int
+ompi_mpi_cxx_grequest_free_fn_intercept(void *state)
+{
+    struct MPI::Grequest_intercept_t *data = 
+        (struct MPI::Grequest_intercept_t *) state;
+    int ret = data->git_cxx_free_fn(data->git_extra);
+    // Delete the struct that was "new"ed in MPI::Grequest::Start()
+    delete data;
+    return ret;
+}
+
+extern "C" int
+ompi_mpi_cxx_grequest_cancel_fn_intercept(void *state, int cancelled)
+{
+    struct MPI::Grequest_intercept_t *data = 
+        (struct MPI::Grequest_intercept_t *) state;
+    return data->git_cxx_cancel_fn(data->git_extra, (bool) cancelled);
+}

ompi/mpi/cxx/mpicxx.h

@@ -10,6 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
+// Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -90,6 +91,17 @@ extern "C" int
 ompi_mpi_cxx_delete_attr_intercept(MPI_Comm comm, int keyval, 
                                    void *attribute_val, void *extra_state);
 
+//
+// MPI generalized request intercepts
+//
+
+extern "C" int 
+ompi_mpi_cxx_grequest_query_fn_intercept(void *state, MPI_Status *status);
+extern "C" int 
+ompi_mpi_cxx_grequest_free_fn_intercept(void *state);
+extern "C" int 
+ompi_mpi_cxx_grequest_cancel_fn_intercept(void *state, int canceled);
+
 /**
  * Windows bool type is not any kind of integer. Special care should
  * be taken in order to cast it correctly.

ompi/mpi/cxx/request.h

@@ -192,6 +192,10 @@ private:
 #endif 
 };
 
+
+//
+// Generalized requests
+//
 class Grequest : public MPI::Request {
   public:
     typedef int Query_function(void *, Status&);
@@ -216,3 +220,15 @@ class Grequest : public MPI::Request {
 
     virtual void Complete();
 };
+
+//
+// Type used for intercepting Generalized requests in the C++ layer so
+// that the type can be converted to C++ types before invoking the
+// user-specified C++ callbacks.
+//
+struct Grequest_intercept_t {
+    void *git_extra;
+    Grequest::Query_function *git_cxx_query_fn;
+    Grequest::Free_function *git_cxx_free_fn;
+    Grequest::Cancel_function *git_cxx_cancel_fn;
+};

ompi/mpi/cxx/request_inln.h

@@ -10,6 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
+// Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -315,6 +316,9 @@ inline bool MPI::Request::Get_status(MPI::Status& status) const
     int flag;
     MPI_Status c_status;
 
+    // Call the underlying MPI function rather than simply returning
+    // status.mpi_status because we may have to invoke the generalized
+    // request query function
     (void)MPI_Request_get_status(mpi_request, &flag, &c_status);
     if (flag) {
         status = c_status;
@@ -326,6 +330,9 @@ inline bool MPI::Request::Get_status() const
 {
     int flag;
 
+    // Call the underlying MPI function rather than simply returning
+    // status.mpi_status because we may have to invoke the generalized
+    // request query function
     (void)MPI_Request_get_status(mpi_request, &flag, MPI_STATUS_IGNORE);
     return OPAL_INT_TO_BOOL(flag);
 }
@@ -335,10 +342,17 @@ MPI::Grequest::Start(Query_function *query_fn, Free_function *free_fn,
 	Cancel_function *cancel_fn, void *extra)
 {
     MPI_Request grequest = 0;
+    struct Grequest_intercept_t *new_extra = 
+        new struct MPI::Grequest_intercept_t;
 
-    (void) MPI_Grequest_start((MPI_Grequest_query_function *) query_fn,
-	    (MPI_Grequest_free_function *) free_fn,
-	    (MPI_Grequest_cancel_function *) cancel_fn, extra, &grequest);
+    new_extra->git_extra = extra;
+    new_extra->git_cxx_query_fn = query_fn;
+    new_extra->git_cxx_free_fn = free_fn;
+    new_extra->git_cxx_cancel_fn = cancel_fn;
+    (void) MPI_Grequest_start(ompi_mpi_cxx_grequest_query_fn_intercept,
+                              ompi_mpi_cxx_grequest_free_fn_intercept,
+                              ompi_mpi_cxx_grequest_cancel_fn_intercept,
+                              new_extra, &grequest);
 
     return(grequest);
 }

ompi/mpi/cxx/status.h

@@ -10,6 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
+// Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -98,6 +99,10 @@ public:
 
   virtual void Set_error(int error);
 
+  virtual void Set_elements(const MPI::Datatype& datatype, int count);
+
+  virtual void Set_cancelled(bool flag);
+
 protected:
 #if 0 /* OMPI_ENABLE_MPI_PROFILING */
   PMPI::Status pmpi_status;

ompi/mpi/cxx/status_inln.h

@@ -10,6 +10,7 @@
 //                         University of Stuttgart.  All rights reserved.
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
+// Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
 // $COPYRIGHT$
 // 
 // Additional copyrights may follow
@@ -89,3 +90,16 @@ MPI::Status::Set_error(int error)
 {
   mpi_status.MPI_ERROR = error;
 }
+
+inline void
+MPI::Status::Set_elements(const MPI::Datatype& datatype, int count)
+{
+    MPI_Status_set_elements(&mpi_status, datatype, count);
+}
+
+inline void
+MPI::Status::Set_cancelled(bool flag)
+{
+    mpi_status._cancelled = (int) flag;
+}
+

ompi/request/grequest.h

@@ -28,33 +28,51 @@ extern "C" {
 #endif
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_grequest_t);
 
-/*
- * Fortran types for the generalized request functions.
+/**
+ * Fortran type for generalized request query function
  */
 typedef void (MPI_F_Grequest_query_function)(MPI_Aint *extra_state, 
                                              MPI_Fint *status, 
                                              MPI_Fint *ierr);
+/**
+ * Fortran type for generalized request free function
+ */
 typedef void (MPI_F_Grequest_free_function)(MPI_Aint *extra_state, 
                                             MPI_Fint *ierr);
+/**
+ * Fortran type for generalized request cancel function
+ */
 typedef void (MPI_F_Grequest_cancel_function)(MPI_Aint *extra_state, 
                                               MPI_Flogical *complete, 
                                               MPI_Fint *ierr);
 
+/**
+ * Union for query function for use in ompi_grequest_t
+ */
 typedef union {
     MPI_Grequest_query_function*   c_query;
     MPI_F_Grequest_query_function* f_query;
 } MPI_Grequest_query_fct_t;
 
+/**
+ * Union for free function for use in ompi_grequest_t
+ */
 typedef union {
     MPI_Grequest_free_function*   c_free;
     MPI_F_Grequest_free_function* f_free;
 } MPI_Grequest_free_fct_t;
 
+/**
+ * Union for cancel function for use in ompi_grequest_t
+ */
 typedef union {
     MPI_Grequest_cancel_function*   c_cancel;
     MPI_F_Grequest_cancel_function* f_cancel;
 } MPI_Grequest_cancel_fct_t;
 
+/**
+ * Main structure for MPI generalized requests
+ */
 struct ompi_grequest_t {
     ompi_request_t greq_base;
     MPI_Grequest_query_fct_t greq_query;
@@ -63,11 +81,13 @@ struct ompi_grequest_t {
     void *greq_state;
     bool greq_funcs_are_c;
 };
+/**
+ * Convenience typedef
+ */
 typedef struct ompi_grequest_t ompi_grequest_t;
 
-
-/*
- * Start a generalized request.
+/**
+ * Start a generalized request (back end for MPI_GREQUEST_START)
  */
 OMPI_DECLSPEC int ompi_grequest_start(
     MPI_Grequest_query_function *gquery,
@@ -76,12 +96,12 @@ OMPI_DECLSPEC int ompi_grequest_start(
     void* gstate,
     ompi_request_t** request);
 
-/*
- * Complete a generalized request
+/**
+ * Complete a generalized request (back end for MPI_GREQUEST_COMPLETE)
  */
 OMPI_DECLSPEC int ompi_grequest_complete(ompi_request_t *req);
 
-/*
+/**
  * Invoke the query function on a generalized request
  */
 OMPI_DECLSPEC int ompi_grequest_invoke_query(ompi_request_t *request,