Currently MTLs do no handle communicator contexts in any special way,

they only add the context id to the tag selection of the underlying 
messaging meachinsm. 
 
 We would like to enable an MTL to maintain its own context data
per-communicator. This way an MTL will be able to queue incoming eager 
messages and rendezvous requests per-communicator basis.

 The MTL will be allowed to override comm->c_pml_comm member, 
since it's unused in pml_cm anyway. 

This commit was SVN r24858.

ompi/mca/mtl/mtl.h

@@ -354,6 +354,34 @@ typedef int (*mca_mtl_base_module_cancel_fn_t)(
                           int flag);
 
 
+/**
+ * Downcall from PML layer when a new communicator is created.
+ *
+ * @param comm  Communicator
+ * @return      OMPI_SUCCESS or failure status.
+ *
+ * Provides the MTL the opportunity to initialize/cache a data structure
+ * on the communicator.
+ */
+typedef int (*mca_mtl_base_module_add_comm_fn_t)(
+                          struct mca_mtl_base_module_t* mtl,
+                          struct ompi_communicator_t* comm);
+
+
+/**
+ * Downcall from PML layer when a communicator is destroyed.
+ *
+ * @param comm  Communicator
+ * @return      OMPI_SUCCESS or failure status.
+ *
+ * Provides the MTL the opportunity to cleanup any datastructures
+ * associated with the communicator.
+ */
+typedef int (*mca_mtl_base_module_del_comm_fn_t)(
+                          struct mca_mtl_base_module_t* mtl,
+                          struct ompi_communicator_t* comm);
+
+
 /**
  * MTL module interface functions and attributes.
  */
@@ -376,6 +404,8 @@ struct mca_mtl_base_module_t {
 
     /* Optional MTL functions */
     mca_mtl_base_module_cancel_fn_t      mtl_cancel;
+    mca_mtl_base_module_add_comm_fn_t    mtl_add_comm;
+    mca_mtl_base_module_del_comm_fn_t    mtl_del_comm;
 };
 typedef struct mca_mtl_base_module_t mca_mtl_base_module_t;
 

ompi/mca/mtl/mx/mtl_mx.c

@@ -59,7 +59,9 @@ mca_mtl_mx_module_t ompi_mtl_mx = {
         ompi_mtl_mx_irecv,
         ompi_mtl_mx_iprobe,
         
-        ompi_mtl_mx_cancel
+        ompi_mtl_mx_cancel,
+        NULL,
+        NULL
     }    
 };
 

ompi/mca/mtl/portals/mtl_portals.c

@@ -53,7 +53,9 @@ mca_mtl_portals_module_t ompi_mtl_portals = {
         ompi_mtl_portals_irecv,
         ompi_mtl_portals_iprobe,
 
-        NULL        /* cancel */
+        NULL,       /* cancel */
+        NULL,       /* add_comm */
+        NULL        /* del_comm */
     }
 };
 

ompi/mca/mtl/portals4/mtl_portals4.c

@@ -50,7 +50,9 @@ mca_mtl_portals4_module_t ompi_mtl_portals4 = {
         ompi_mtl_portals4_irecv,
         ompi_mtl_portals4_iprobe,
 
-        NULL        /* cancel */
+        NULL,       /* cancel */
+        NULL,       /* add_comm */
+        NULL        /* del_comm */
     }
 };
 

ompi/mca/mtl/psm/mtl_psm.c

@@ -48,7 +48,9 @@ mca_mtl_psm_module_t ompi_mtl_psm = {
         ompi_mtl_psm_irecv,
         ompi_mtl_psm_iprobe,
         
-        ompi_mtl_psm_cancel
+        ompi_mtl_psm_cancel,
+        NULL,
+        NULL
     }    
 };
 

ompi/mca/pml/cm/pml_cm.c

@@ -82,14 +82,22 @@ mca_pml_cm_enable(bool enable)
 int
 mca_pml_cm_add_comm(ompi_communicator_t* comm)
 {
+    int ret;
+
     /* should never happen, but it was, so check */
     if (comm->c_contextid > ompi_pml_cm.super.pml_max_contextid) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    /* setup our per-communicator data */
+    /* initialize per-communicator data. MTLs may override this. */
     comm->c_pml_comm = NULL;
 
+    /* notify the MTL about the added communicator */
+    if ((NULL != ompi_mtl->mtl_add_comm) &&
+        (OMPI_SUCCESS != (ret = OMPI_MTL_CALL(add_comm(ompi_mtl, comm))))) {
+        return ret;
+    }
+
     return OMPI_SUCCESS;
 }
 
@@ -97,8 +105,13 @@ mca_pml_cm_add_comm(ompi_communicator_t* comm)
 int
 mca_pml_cm_del_comm(ompi_communicator_t* comm)
 {
-    /* clean up our per-communicator data */
-    comm->c_pml_comm = NULL;
+    int ret;
+
+    /* notify the MTL about the deleted communicator */
+    if ((NULL != ompi_mtl->mtl_del_comm) &&
+        (OMPI_SUCCESS != (ret = OMPI_MTL_CALL(del_comm(ompi_mtl, comm))))) {
+        return ret;
+    }
 
     return OMPI_SUCCESS;
 }