Allow MX to handle shared memory and self communications. By default these features

are disabled (btl_mx_shared_mem respectively btl_mx_self have to be set in order
to activate them).

This commit was SVN r12922.

ompi/mca/btl/mx/btl_mx.c

@@ -79,15 +79,20 @@ int mca_btl_mx_add_procs( struct mca_btl_base_module_t* btl,
         mca_btl_mx_proc_t* mx_proc;
         mca_btl_base_endpoint_t* mx_endpoint;
 
-        /* We have special BTLs for processes on the same node as well as for all communications
-         * inside the same process. Therefore, MX will not be used for any of them.
-         */
-        if( (ompi_procs[i] == ompi_proc_local_proc) ||
-            ( (0 == mca_btl_mx_component.mx_support_sharedmem) &&
-              (ompi_procs[i]->proc_flags & OMPI_PROC_FLAG_LOCAL) ) ) {
-            continue;
+        /**
+         * By default don't allow communications with itself nor
+         * with any other processes on the same node.
+         * BTL self and sm are supposed to take care of such communications.
+          */
+        if( ompi_procs[i]->proc_flags & OMPI_PROC_FLAG_LOCAL ) {
+            if( ompi_procs[i] == ompi_proc_local_proc ) {
+                if( 0 == mca_btl_mx_component.mx_support_self )
+                    continue;
+            } else {
+                if( 0 == mca_btl_mx_component.mx_support_sharedmem )
+                    continue;
+            }
         }
-
         if( NULL == (mx_proc = mca_btl_mx_proc_create(ompi_proc)) ) {
             continue;
         }

ompi/mca/btl/mx/btl_mx.h

@@ -71,6 +71,8 @@ struct mca_btl_mx_component_t {
 
     int32_t                                 mx_support_sharedmem;
     /**< true if we want to activate the MX support for shared memory */
+    int32_t                                 mx_support_self;
+    /**< true if we want to activate the MX support for self communications */
 
     opal_list_t mx_procs;  /**< list of mx proc structures */
 

ompi/mca/btl/mx/btl_mx_component.c

@@ -98,9 +98,12 @@ int mca_btl_mx_component_open(void)
     mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "filter",
                             "Unique ID for the application (used to connect to the peers)",
                             false, false, 0xdeadbeef, &mca_btl_mx_component.mx_filter );
+    mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "self",
+                            "Enable the MX support for self communications",
+                            false, false, 0, &mca_btl_mx_component.mx_support_self );
     mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "shared_mem",
                             "Enable the MX support for shared memory",
-                            false, true, 0, &mca_btl_mx_component.mx_support_sharedmem );
+                            false, false, 0, &mca_btl_mx_component.mx_support_sharedmem );
 
     mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "free_list_num",
                             "Number of allocated default request",
@@ -150,7 +153,8 @@ int mca_btl_mx_component_open(void)
     mca_btl_mx_module.super.btl_max_rdma_size = tmp;
     mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "flags",
                             "Flags to activate/deactivate the RDMA",
-                            true, false, MCA_BTL_FLAGS_PUT, (int*)&mca_btl_mx_module.super.btl_flags );
+                            true, false, MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT,
+                            (int*)&mca_btl_mx_module.super.btl_flags );
 
     return OMPI_SUCCESS;
 }
@@ -197,7 +201,6 @@ static mca_btl_mx_module_t* mca_btl_mx_create(uint64_t addr)
 
     /* copy over default settings */
     memcpy( mx_btl, &mca_btl_mx_module, sizeof(mca_btl_mx_module_t) );
-    mx_btl->super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE; /* | MCA_BTL_FLAGS_PUT;*/
     OBJ_CONSTRUCT( &mx_btl->mx_peers, opal_list_t );
     OBJ_CONSTRUCT( &mx_btl->mx_lock, opal_mutex_t );
     /* open local endpoint */
@@ -252,9 +255,10 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
      * up and running the MX library does not exit the application.
      */
     mx_set_error_handler(MX_ERRORS_RETURN);
-    /* Until this BTL reach a stable state let MX library generate assert for the errors */
-    /*mx_set_error_handler(MX_ERRORS_ARE_FATAL);*/
-    opal_setenv( "MX_DISABLE_SHMEM", "1", true, &environ );
+    if( 0 == mca_btl_mx_component.mx_support_sharedmem )
+        opal_setenv( "MX_DISABLE_SHMEM", "1", true, &environ );
+    if( 0 == mca_btl_mx_component.mx_support_self )
+        opal_setenv( "MX_DISABLE_SELF", "1", true, &environ );
 
     /* First check if MX is available ... */
     if( MX_SUCCESS != (status = mx_init()) ) {

ompi/mca/btl/mx/btl_mx_frag.c

@@ -20,10 +20,9 @@
 
 static void mca_btl_mx_frag_common_constructor(mca_btl_mx_frag_t* frag) 
 { 
-    mca_btl_mx_frag_common_constructor(frag); 
-    frag->base.des_src = NULL;
+    frag->base.des_src     = NULL;
     frag->base.des_src_cnt = 0;
-    frag->base.des_dst = NULL;
+    frag->base.des_dst     = NULL;
     frag->base.des_dst_cnt = 0;
 }
 

ompi/mca/btl/mx/btl_mx_proc.c

@@ -120,7 +120,7 @@ mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc)
 
     module_proc = OBJ_NEW(mca_btl_mx_proc_t);
 
-    module_proc->proc_ompi      = ompi_proc;
+    module_proc->proc_ompi = ompi_proc;
 
     return module_proc;
 }