make portals level ack's optional (require ACK by default)

This commit was SVN r18228.

ompi/mca/btl/portals/btl_portals.h

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -63,6 +64,10 @@ struct mca_btl_portals_component_t {
     
     /* shall I use portals to send to thyself? */
     int portals_support_self;
+
+    /* do I need a portals ACK? */
+    int portals_need_ack;
+
 };
 typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;
 

ompi/mca/btl/portals/btl_portals_component.c

@@ -125,7 +125,7 @@ mca_btl_portals_component_open(void)
                            false,
                            32,
                            &(mca_btl_portals_component.portals_free_list_eager_max_num));
-
+    
     mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
                            "support_self",
                            "Use portals for send to self",
@@ -134,6 +134,14 @@ mca_btl_portals_component_open(void)
                            1, /* default to true.. */ 
                            &(mca_btl_portals_component.portals_support_self));
 
+    mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version,
+                           "needs_ack",
+                           "Require a portals level ACK",
+                           false,
+                           false,
+                           1, /* default to true.. */ 
+                           &(mca_btl_portals_component.portals_need_ack));
+
     /* 
      * fill default module state 
      */
@@ -490,7 +498,7 @@ mca_btl_portals_component_progress(void)
                 OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
                                      "PTL_EVENT_SEND_END for 0x%lx, %d",
                                      (unsigned long) frag, (int) ev.hdr_data));
-
+                
                 if (ev.ni_fail_type != PTL_NI_OK) {
                     opal_output(mca_btl_portals_component.portals_output,
                                 "Failure to end send event\n");
@@ -504,6 +512,22 @@ mca_btl_portals_component_progress(void)
                     }
                 }
 #endif
+                if(!mca_btl_portals_component.portals_need_ack) { 
+                    /* my part's done, in portals we trust! */
+                    frag->base.des_cbfunc(&mca_btl_portals_module.super,
+                                          frag->endpoint,
+                                          &frag->base,
+                                          OMPI_SUCCESS);
+                    if( btl_ownership ) {
+                        mca_btl_portals_free(&mca_btl_portals_module.super,
+                                             &frag->base);
+                    }
+                    if (0 != frag->size) {
+                        OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops,
+                                          -1);
+                        MCA_BTL_PORTALS_PROGRESS_QUEUED_SENDS();
+                    }
+                }
                 break;
 
             case PTL_EVENT_ACK:
@@ -516,8 +540,13 @@ mca_btl_portals_component_progress(void)
                 OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
                                      "PTL_EVENT_ACK for 0x%lx",
                                      (unsigned long) frag));
-
+                
 #if OMPI_ENABLE_DEBUG
+                if(!mca_btl_portals_component.portals_need_ack) { 
+                    opal_output(mca_btl_portals_component.portals_output, 
+                                "Received PTL_EVENT_ACK but ACK's are disabled!\n");
+                    abort();
+                }
                 if (ev.ni_fail_type != PTL_NI_OK) {
                     opal_output(mca_btl_portals_component.portals_output,
                                 "Failure to ack event\n");

ompi/mca/btl/portals/btl_portals_rdma.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -48,9 +49,9 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
 
     /* setup the send */
     assert(1 == frag->base.des_src_cnt);
-
+    
     ret = PtlPut(frag->md_h,
-                 PTL_ACK_REQ,
+                 (mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
                  *((mca_btl_base_endpoint_t*) btl_peer),
                  OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                  0, /* ac_index - not used*/

ompi/mca/btl/portals/btl_portals_send.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -62,8 +63,16 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
         mca_btl_portals_module.md_send.start = frag->segments[0].seg_addr.pval;
         mca_btl_portals_module.md_send.length = 
             0 == frag->size ? frag->segments[0].seg_len : frag->size;
+#if OMPI_ENABLE_DEBUG 
         mca_btl_portals_module.md_send.options = 
             PTL_MD_EVENT_START_DISABLE;
+#else 
+        /* optimized build, we can get rid of the END event */
+        /*  if we are using portals ACK's for completion */
+        mca_btl_portals_module.md_send.options = 
+            (PTL_MD_EVENT_START_DISABLE | 
+             (mca_btl_portals_component.portals_need_ack ? PTL_MD_EVENT_END_DISABLE : 0));
+#endif
         mca_btl_portals_module.md_send.user_ptr = frag; /* keep a pointer to ourselves */
 
         /* make a free-floater */
@@ -84,11 +93,11 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
                          "\tlen: %d",
                          (unsigned long) frag->segments[0].seg_addr.pval,
                          frag->segments[0].seg_len)); 
-
+    
     ret = PtlPutRegion(frag->md_h,                /* memory descriptor */
                        0,                         /* fragment offset */
                        frag->segments[0].seg_len, /* fragment length */
-                       PTL_ACK_REQ,
+                       (mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
                        *((mca_btl_base_endpoint_t*) endpoint),
                        OMPI_BTL_PORTALS_SEND_TABLE_ID,
                        0,                         /* ac_index - not used */