* Clean up error checking in the MPI interface for MPI-2 onesided

* Implement fortran handle -> c handle tracking
* Remove some unneeded locking around free lists (the free list
  macros do their own locking)
* Try to be a bit more memory friendly with the w_mode setting /
  checking

This commit was SVN r8865.

ompi/mca/osc/pt2pt/osc_pt2pt.c

@@ -33,7 +33,6 @@ ompi_osc_pt2pt_module_free(ompi_win_t *win)
     int i, tmp;
     ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);
 
-    /* are we in an epoch? */
     if ((OMPI_WIN_ACCESS_EPOCH & win->w_flags) || 
         (OMPI_WIN_EXPOSE_EPOCH & win->w_flags)) {
         /* finish off the epoch.  More for sanity checks than anything

ompi/mca/osc/pt2pt/osc_pt2pt.h

@@ -67,9 +67,6 @@ struct ompi_osc_pt2pt_module_t {
     /** communicator created with this window */
     ompi_communicator_t *p2p_comm;
 
-    /** store weather user disabled locks for this window */
-    bool p2p_want_locks;
-
     /** array of opal_list_ts, not a pointer to one of them.  Array is
         of size <num ranks in communicator>, although only the first
         <num ranks in group> are used for PWSC synchronization */

ompi/mca/osc/pt2pt/osc_pt2pt_component.c

@@ -180,21 +180,7 @@ ompi_osc_pt2pt_component_query(ompi_win_t *win,
                               ompi_info_t *info,
                               ompi_communicator_t *comm)
 {
-    if (!mca_osc_pt2pt_component.p2p_c_have_progress_threads) {
+    /* we can always run - return a low priority */
-        /* if we don't have threads, we can only run if the user
-           promises not to use locking by setting the no_locks key */
-#if 0
-        if (want_locks(info)) { 
-            /* once the default build of Open MPI is to use progress
-               threads, should enable this check again.  For now,
-               though, we just abort when we get to the call to
-               MPI_Lock() */
-            return -1
-        }
-#endif
-    }
-
-    /* woo!  we can run! Return priority of 10 (low) */
     return 10;
 }
 
@@ -229,10 +215,7 @@ ompi_osc_pt2pt_component_select(ompi_win_t *win,
         return ret;
     }
 
-    module->p2p_want_locks = want_locks(info);
+    if (!want_locks(info)) win->w_flags |= OMPI_WIN_NO_LOCKS;
-    if (!mca_osc_pt2pt_component.p2p_c_have_progress_threads) {
-        module->p2p_want_locks = false;
-    }
 
     module->p2p_pending_out_sendreqs = malloc(sizeof(opal_list_t) *
                                               ompi_comm_size(module->p2p_comm));
@@ -269,6 +252,9 @@ ompi_osc_pt2pt_component_select(ompi_win_t *win,
                                ompi_osc_pt2pt_component_fragment_cb,
                                NULL);
 
+    /* sync memory - make sure all initialization completed */
+    opal_atomic_mb();
+
     return ret;
 }
 
@@ -285,7 +271,8 @@ ompi_osc_pt2pt_component_fragment_cb(struct mca_btl_base_module_t *btl,
     ompi_osc_pt2pt_module_t *module;
     void *payload;
 
-    assert(descriptor->des_dst[0].seg_len >= sizeof(ompi_osc_pt2pt_type_header_t));
+    assert(descriptor->des_dst[0].seg_len >= 
+           sizeof(ompi_osc_pt2pt_type_header_t));
 
     /* handle message */
     switch (((ompi_osc_pt2pt_type_header_t*) descriptor->des_dst[0].seg_addr.pval)->hdr_type) {

ompi/mca/osc/pt2pt/osc_pt2pt_header.h

@@ -59,12 +59,20 @@ struct ompi_osc_pt2pt_control_header_t {
 };
 typedef struct ompi_osc_pt2pt_control_header_t ompi_osc_pt2pt_control_header_t;
 
+struct ompi_osc_pt2pt_lock_header_t {
+    uint8_t hdr_type;
+    int32_t hdr_windx;
+    int32_t hdr_origin;
+};
+typedef struct ompi_osc_pt2pt_lock_header_t ompi_osc_pt2pt_lock_header_t;
+
 #define OMPI_OSC_PT2PT_HDR_PUT      0x0001
 #define OMPI_OSC_PT2PT_HDR_ACC      0x0002
 #define OMPI_OSC_PT2PT_HDR_GET      0x0004
 #define OMPI_OSC_PT2PT_HDR_REPLY    0x0008
 #define OMPI_OSC_PT2PT_HDR_POST     0x0010
 #define OMPI_OSC_PT2PT_HDR_COMPLETE 0x0020
+#define OMPI_OSC_PT2PT_HDR_LOCK     0x0040
 
 /*
  * Convert a 64 bit value to network byte order.

ompi/mca/osc/pt2pt/osc_pt2pt_longreq.h

@@ -53,10 +53,8 @@ ompi_osc_pt2pt_longreq_alloc(ompi_osc_pt2pt_longreq_t **longreq)
     opal_list_item_t *item;
     int ret;
 
-    OPAL_THREAD_LOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     OPAL_FREE_LIST_GET(&mca_osc_pt2pt_component.p2p_c_longreqs,
                        item, ret);
-    OPAL_THREAD_UNLOCK(&mca_osc_pt2pt_component.p2p_c_lock);
 
     *longreq = (ompi_osc_pt2pt_longreq_t*) item;
     return ret;
@@ -65,10 +63,8 @@ ompi_osc_pt2pt_longreq_alloc(ompi_osc_pt2pt_longreq_t **longreq)
 static inline int
 ompi_osc_pt2pt_longreq_free(ompi_osc_pt2pt_longreq_t *longreq)
 {
-    OPAL_THREAD_LOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     OPAL_FREE_LIST_RETURN(&mca_osc_pt2pt_component.p2p_c_longreqs,
                           (opal_list_item_t*) longreq);
-    OPAL_THREAD_UNLOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     return OMPI_SUCCESS;
 }
 

ompi/mca/osc/pt2pt/osc_pt2pt_replyreq.h

@@ -78,10 +78,8 @@ ompi_osc_pt2pt_replyreq_alloc(ompi_osc_pt2pt_module_t *module,
     /* BWB - FIX ME - is this really the right return code? */
     if (NULL == proc) return OMPI_ERR_OUT_OF_RESOURCE;
 
-    OPAL_THREAD_LOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     OPAL_FREE_LIST_GET(&mca_osc_pt2pt_component.p2p_c_replyreqs,
                        item, ret);
-    OPAL_THREAD_UNLOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     if (OMPI_SUCCESS != ret) return ret;
     *replyreq = (ompi_osc_pt2pt_replyreq_t*) item;
 
@@ -131,10 +129,8 @@ ompi_osc_pt2pt_replyreq_free(ompi_osc_pt2pt_replyreq_t *replyreq)
 
     OBJ_RELEASE(replyreq->rep_target_datatype);
 
-    OPAL_THREAD_LOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     OPAL_FREE_LIST_RETURN(&mca_osc_pt2pt_component.p2p_c_replyreqs,
                           (opal_list_item_t*) replyreq);
-    OPAL_THREAD_UNLOCK(&mca_osc_pt2pt_component.p2p_c_lock);
  
     return OMPI_SUCCESS;
 }

ompi/mca/osc/pt2pt/osc_pt2pt_sendreq.h

@@ -94,10 +94,8 @@ ompi_osc_pt2pt_sendreq_alloc(ompi_osc_pt2pt_module_t *module,
     /* BWB - FIX ME - is this really the right return code? */
     if (NULL == proc) return OMPI_ERR_OUT_OF_RESOURCE;
 
-    OPAL_THREAD_LOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     OPAL_FREE_LIST_GET(&mca_osc_pt2pt_component.p2p_c_sendreqs,
                        item, ret);
-    OPAL_THREAD_UNLOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     if (OMPI_SUCCESS != ret) return ret;
     *sendreq = (ompi_osc_pt2pt_sendreq_t*) item;
 
@@ -166,10 +164,8 @@ ompi_osc_pt2pt_sendreq_free(ompi_osc_pt2pt_sendreq_t *sendreq)
     OBJ_RELEASE(sendreq->req_target_datatype);
     OBJ_RELEASE(sendreq->req_origin_datatype);
 
-    OPAL_THREAD_LOCK(&mca_osc_pt2pt_component.p2p_c_lock);
     OPAL_FREE_LIST_RETURN(&mca_osc_pt2pt_component.p2p_c_sendreqs,
                           (opal_list_item_t*) sendreq);
-    OPAL_THREAD_UNLOCK(&mca_osc_pt2pt_component.p2p_c_lock);
  
     return OMPI_SUCCESS;
 }

ompi/mca/osc/pt2pt/osc_pt2pt_sync.c

@@ -28,13 +28,13 @@
 #include "ompi/communicator/communicator.h"
 
 
+/* should have p2p_lock before calling */
 static inline void
 ompi_osc_pt2pt_progress(ompi_osc_pt2pt_module_t *module)
 {
     if (0 != module->p2p_num_long_msgs) {
         opal_list_item_t *item, *next;
 
-        OPAL_THREAD_LOCK(&(module->p2p_lock));
         /* Have to go the convoluted while() route instead of a for()
            loop because the callback will likely remove the request
            from the list and free it, and that would lead to much
@@ -52,7 +52,6 @@ ompi_osc_pt2pt_progress(ompi_osc_pt2pt_module_t *module)
                 longreq->req_comp_cb(longreq);
             }
         }
-        OPAL_THREAD_UNLOCK(&(module->p2p_lock));
     }
     opal_progress();
 }
@@ -155,9 +154,9 @@ ompi_osc_pt2pt_module_fence(int assert, ompi_win_t *win)
 
     /* all transfers are done - back to the real world we go */
     if (0 == (assert & MPI_MODE_NOSUCCEED)) {
-        win->w_mode = OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_EXPOSE_EPOCH;
+        ompi_win_set_mode(win, OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_EXPOSE_EPOCH);
     } else {
-        win->w_mode = 0;
+        ompi_win_set_mode(win, 0);
     }
 
  cleanup:
@@ -183,11 +182,10 @@ ompi_osc_pt2pt_module_start(ompi_group_t *group,
     P2P_MODULE(win)->sc_group = group;    
 
     /* Set our mode to access w/ start */
-    win->w_mode = OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_STARTED;
+    ompi_win_set_mode(win, OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_STARTED);
 
     /* possible we've already received a couple in messages, so
        atomicall add however many we're going to wait for */
-    assert(P2P_MODULE(win)->p2p_num_pending_in == 0);
     OPAL_THREAD_ADD32(&(P2P_MODULE(win)->p2p_num_pending_in),
                       ompi_group_size(P2P_MODULE(win)->sc_group));
 
@@ -265,7 +263,7 @@ ompi_osc_pt2pt_module_complete(ompi_win_t *win)
 
  cleanup:
     /* set our mode back to nothing */
-    win->w_mode = 0;
+    ompi_win_set_mode(win, 0);
 
     /* BWB - do I need this? */
     ompi_group_decrement_proc_count(P2P_MODULE(win)->sc_group);
@@ -292,7 +290,7 @@ ompi_osc_pt2pt_module_post(ompi_group_t *group,
     P2P_MODULE(win)->pw_group = group;    
 
     /* Set our mode to expose w/ post */
-    win->w_mode = OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED;
+    ompi_win_set_mode(win, OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED);
 
     /* list how many complete counters we're still waiting on */
     OPAL_THREAD_ADD32(&(P2P_MODULE(win)->p2p_num_pending_out),
@@ -320,7 +318,7 @@ ompi_osc_pt2pt_module_wait(ompi_win_t *win)
     }
 
     OPAL_THREAD_LOCK(&(P2P_MODULE(win)->p2p_lock));
-    win->w_mode = 0;
+    ompi_win_set_mode(win, 0);
 
     /* BWB - do I need this? */
     ompi_group_decrement_proc_count(P2P_MODULE(win)->pw_group);
@@ -350,7 +348,7 @@ ompi_osc_pt2pt_module_test(ompi_win_t *win,
     *flag = 1;
 
     OPAL_THREAD_LOCK(&(P2P_MODULE(win)->p2p_lock));
-    win->w_mode = 0;
+    ompi_win_set_mode(win, 0);
 
     /* BWB - do I need this? */
     ompi_group_decrement_proc_count(P2P_MODULE(win)->pw_group);
@@ -365,21 +363,17 @@ ompi_osc_pt2pt_module_test(ompi_win_t *win,
 
 int
 ompi_osc_pt2pt_module_lock(int lock_type,
-                               int target,
+                           int target,
-                               int assert,
+                           int assert,
-                               ompi_win_t *win)
+                           ompi_win_t *win)
 {
-    if (!P2P_MODULE(win)->p2p_want_locks) {
-        return MPI_ERR_OTHER;
-    }
-
     return OMPI_ERR_NOT_IMPLEMENTED;
 }
 
 
 int
 ompi_osc_pt2pt_module_unlock(int target,
-                                 ompi_win_t *win)
+                             ompi_win_t *win)
 {
     return OMPI_ERR_NOT_IMPLEMENTED;
 }

ompi/mpi/c/accumulate.c

@@ -61,8 +61,7 @@ int MPI_Accumulate(void *origin_addr, int origin_count, MPI_Datatype origin_data
                    (target_datatype->id < DT_MAX_PREDEFINED &&
                     -1 == ompi_op_ddt_map[target_datatype->id])) {
             rc = MPI_ERR_OP;
-        } else if (0 == (win->w_mode & OMPI_WIN_ACCESS_EPOCH)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_ACCESS_EPOCH)) {
-            /* BWB - FIX ME - what error? */
             rc = MPI_ERR_RMA_CONFLICT;
         } else {
             OMPI_CHECK_DATATYPE_FOR_SEND(rc, origin_datatype, origin_count);

ompi/mpi/c/get.c

@@ -53,8 +53,7 @@ int MPI_Get(void *origin_addr, int origin_count,
             rc = MPI_ERR_COUNT;
         } else if (ompi_win_peer_invalid(win, target_rank)) {
             rc = MPI_ERR_RANK;
-        } else if (0 == (win->w_mode & OMPI_WIN_ACCESS_EPOCH)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_ACCESS_EPOCH)) {
-            /* BWB - FIX ME - what error? */
             rc = MPI_ERR_RMA_CONFLICT;
         } else {
             OMPI_CHECK_DATATYPE_FOR_SEND(rc, origin_datatype, origin_count);

ompi/mpi/c/put.c

@@ -52,8 +52,7 @@ int MPI_Put(void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
             rc = MPI_ERR_COUNT;
         } else if (ompi_win_peer_invalid(win, target_rank)) {
             rc = MPI_ERR_RANK;
-        } else if (0 == (win->w_mode & OMPI_WIN_ACCESS_EPOCH)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_ACCESS_EPOCH)) {
-            /* BWB - FIX ME - what error? */
             rc = MPI_ERR_RMA_CONFLICT;
         } else {
             OMPI_CHECK_DATATYPE_FOR_SEND(rc, origin_datatype, origin_count);

ompi/mpi/c/win_c2f.c

@@ -16,8 +16,10 @@
  * $HEADER$
  */
 #include "ompi_config.h"
+
 #include <stdio.h>
 
+#include "win/win.h"
 #include "mpi/c/bindings.h"
 #include "mpi/f77/fint_2_int.h"
 
@@ -34,11 +36,13 @@ static const char FUNC_NAME[] = "MPI_Win_c2f";
 
 MPI_Fint MPI_Win_c2f(MPI_Win win) 
 {
-    if (MPI_PARAM_CHECK) {
+    if ( MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            win = MPI_WIN_NULL;
+        }
     }
 
-    /* This function is not yet implemented */
+    return OMPI_INT_2_FINT(win->w_f_to_c_index);
-
-    return (MPI_Fint) OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, FUNC_NAME);
 }

ompi/mpi/c/win_call_errhandler.c

@@ -34,21 +34,18 @@ static const char FUNC_NAME[] = "MPI_Win_call_errhandler";
 
 int MPI_Win_call_errhandler(MPI_Win win, int errorcode) 
 {
-  /* Error checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
 
-  if (MPI_PARAM_CHECK) {
+        if (ompi_win_invalid(win)) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_WIN,
-    if (NULL == win ||
+                                          FUNC_NAME);
-        MPI_WIN_NULL == win) {
+        }
-      return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
-                                    FUNC_NAME);
     }
-  }
 
   /* Invoke the errhandler */
-
   OMPI_ERRHANDLER_INVOKE(win, errorcode, FUNC_NAME);
+
   /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
   return MPI_SUCCESS;
-
 }

ompi/mpi/c/win_complete.c

@@ -42,7 +42,7 @@ int MPI_Win_complete(MPI_Win win)
 
         if (ompi_win_invalid(win)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 == (win->w_mode & OMPI_WIN_STARTED)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_STARTED)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
         }
     }

ompi/mpi/c/win_create.c

@@ -38,8 +38,6 @@ static const char FUNC_NAME[] = "MPI_Win_create";
 int MPI_Win_create(void *base, MPI_Aint size, int disp_unit,
                    MPI_Info info, MPI_Comm comm, MPI_Win *win) 
 {
-    ompi_communicator_t *ompi_comm;
-    ompi_win_t *ompi_win;
     int ret = OMPI_SUCCESS;
 
     /* argument checking */
@@ -59,20 +57,18 @@ int MPI_Win_create(void *base, MPI_Aint size, int disp_unit,
         }
     }
 
-    ompi_comm = (ompi_communicator_t*) comm;
+    /* communicator must be an intracommunicator */
-    if (OMPI_COMM_IS_INTER(ompi_comm)) {
+    if (OMPI_COMM_IS_INTER(comm)) {
-        /* must be an intracommunicator */
         return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
     }
 
-    /*
+    /* create window and return */
-     * Create a shell of a window 
+    ret = ompi_win_create(base, size, disp_unit, comm,
-     */
+                          info, win);
-    ret = ompi_win_create(base, size, disp_unit, ompi_comm,
+    if (OMPI_SUCCESS != ret) {
-                          info, &ompi_win);
+        *win = MPI_WIN_NULL;
-    if (OMPI_SUCCESS != ret) return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-
+    }
-    *win = (MPI_Win) ompi_win;
 
     return OMPI_SUCCESS;
 }

ompi/mpi/c/win_create_errhandler.c

@@ -35,27 +35,24 @@ static const char FUNC_NAME[] = "MPI_Win_create_errhandler";
 int MPI_Win_create_errhandler(MPI_Win_errhandler_fn *function,
                               MPI_Errhandler *errhandler) 
 {
-  int err = MPI_SUCCESS;
+    int err = MPI_SUCCESS;
 
-  /* Error checking */
+    if (MPI_PARAM_CHECK) {
-
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-  if (MPI_PARAM_CHECK) {
+        if (NULL == function || 
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+            NULL == errhandler) {
-    if (NULL == function || 
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
-        NULL == errhandler) {
+                                          FUNC_NAME);
-      return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
+        }
-                                    FUNC_NAME);
     }
-  }
 
-  /* Create and cache the errhandler.  Sets a refcount of 1. */
+    /* Create and cache the errhandler.  Sets a refcount of 1. */
+    *errhandler = 
+        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
+                               (ompi_errhandler_generic_handler_fn_t*) function);
+    if (NULL == *errhandler) {
+        err = MPI_ERR_INTERN;
+    }
 
-  *errhandler = 
+    OMPI_ERRHANDLER_RETURN(err, MPI_COMM_WORLD, MPI_ERR_INTERN, FUNC_NAME);
-    ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
-                          (ompi_errhandler_generic_handler_fn_t*) function);
-  if (NULL == *errhandler) {
-    err = MPI_ERR_INTERN;
-  }
-
-  OMPI_ERRHANDLER_RETURN(err, MPI_COMM_WORLD, MPI_ERR_INTERN, FUNC_NAME);
 }

ompi/mpi/c/win_delete_attr.c

@@ -39,9 +39,9 @@ int MPI_Win_delete_attr(MPI_Win win, int win_keyval)
 
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-	if (MPI_WIN_NULL == win) {
+
-	    return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, 
+        if (ompi_win_invalid(win)) {
-					 FUNC_NAME);
+	    return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
 	}
     }
   

ompi/mpi/c/win_f2c.c

@@ -17,6 +17,7 @@
  */
 #include "ompi_config.h"
 
+#include "win/win.h"
 #include "mpi/c/bindings.h"
 #include "mpi/f77/fint_2_int.h"
 
@@ -33,15 +34,19 @@ static const char FUNC_NAME[] = "MPI_Win_f2c";
 
 MPI_Win MPI_Win_f2c(MPI_Fint win) 
 {
+    int o_index= OMPI_FINT_2_INT(win);
+
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
     }
 
-    /* NOTE: Be sure to see the other *f2c functions before implementing
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       this one -- they should all be the same! */
+       invalid fortran handle */
 
-    /* This function is not yet implemented */
+    if ( 0 > o_index ||
-
+         o_index >= ompi_pointer_array_get_size(&ompi_mpi_windows)) {
-    (void)OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, FUNC_NAME);
+        return MPI_WIN_NULL;
-    return MPI_WIN_NULL;
+    }
+        
+    return ompi_pointer_array_get_item(&ompi_mpi_windows, o_index);
 }

ompi/mpi/c/win_fence.c

@@ -45,7 +45,8 @@ int MPI_Win_fence(int assert, MPI_Win win)
         } else if (0 != (assert & ~(MPI_MODE_NOSTORE | MPI_MODE_NOPUT | 
                                     MPI_MODE_NOPRECEDE | MPI_MODE_NOSUCCEED))) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        } else if (0 != (win->w_mode & (OMPI_WIN_POSTED | OMPI_WIN_STARTED))) {
+        } else if (0 != (ompi_win_get_mode(win) &
+                         (OMPI_WIN_POSTED | OMPI_WIN_STARTED))) {
             /* If we're in a post or start, we can't be in a fence */
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
         } 

ompi/mpi/c/win_free.c

@@ -34,18 +34,28 @@ static const char FUNC_NAME[] = "MPI_Win_free";
 
 int MPI_Win_free(MPI_Win *win) 
 {
-    int ret;
+    int ret, mode;
 
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
 
-        if (NULL == win) {
+        if (ompi_win_invalid(*win)) {
             return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_WIN, FUNC_NAME);
         }
+
+        /* we allow users to be in an epoch, because that tends to
+           happen with people using MPI_FENCE.  However, do not allow
+           users to be in a Lock / Unlock or a PWSC synchronization */
+        mode = ompi_win_get_mode(win);
+        if (0 != (mode & (OMPI_WIN_POSTED | OMPI_WIN_STARTED | 
+                          OMPI_WIN_LOCK_ACCESS))) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, 
+                                          MPI_ERR_RMA_CONFLICT, 
+                                          FUNC_NAME);
+        }
     }
 
-    /* BWB - fix me - need to add module cleanup code */
+    ret = ompi_win_free(*win);
-    ret = ompi_win_free((ompi_win_t*) *win);
     if (OMPI_SUCCESS == ret) *win = MPI_WIN_NULL;
 
     OMPI_ERRHANDLER_RETURN(ret, *win, ret, FUNC_NAME);

ompi/mpi/c/win_get_attr.c

@@ -40,9 +40,11 @@ int MPI_Win_get_attr(MPI_Win win, int win_keyval,
 
     if (MPI_PARAM_CHECK) {
        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-	if ((NULL == attribute_val) || (NULL == flag)) {
+
-	    return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, 
+        if (ompi_win_invalid(win)) {
-					 FUNC_NAME);
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_WIN, FUNC_NAME);
+        } else if ((NULL == attribute_val) || (NULL == flag)) {
+	    return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
 	}
     }
     

ompi/mpi/c/win_get_errhandler.c

@@ -34,27 +34,22 @@ static const char FUNC_NAME[] = "MPI_Win_get_errhandler";
 
 int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandler) 
 {
-  /* Error checking */
+    if (MPI_PARAM_CHECK) {
-
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-  if (MPI_PARAM_CHECK) {
+        if (ompi_win_invalid(win)) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
-    if (NULL == win || 
+                                          FUNC_NAME);
-        MPI_WIN_NULL == win) {
+        } else if (NULL == errhandler) {
-      return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
-                                    FUNC_NAME);
+                                          FUNC_NAME);
-    } else if (NULL == errhandler) {
+        }
-      return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
-                                    FUNC_NAME);
     }
-  }
 
-  /* Return the errhandler.  See lengthy comment in
+    /* Return the errhandler.  See lengthy comment in
-     comm_get_errhandler.c about why we increment the refcount. */
+       comm_get_errhandler.c about why we increment the refcount. */
+    OBJ_RETAIN(win->error_handler);
+    *errhandler = win->error_handler;
 
-  OBJ_RETAIN(win->error_handler);
+    /* All done */
-  *errhandler = win->error_handler;
+    return MPI_SUCCESS;
-
-  /* All done */
-
-  return MPI_SUCCESS;
 }

ompi/mpi/c/win_get_group.c

@@ -35,19 +35,17 @@ static const char FUNC_NAME[] = "MPI_Win_get_group";
 int MPI_Win_get_group(MPI_Win win, MPI_Group *group) 
 {
     int ret;
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
 
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
 
-        if (MPI_WIN_NULL == ompi_win) {
+        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_INVOKE(ompi_win, MPI_ERR_WIN, FUNC_NAME);
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
-        }
+        } else if (NULL == group) {
-        if (NULL == group) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-            return OMPI_ERRHANDLER_INVOKE(ompi_win, MPI_ERR_ARG, FUNC_NAME);
         }
     }
 
-    ret = ompi_win_group(ompi_win, (ompi_group_t**) group);
+    ret = ompi_win_group(win, (ompi_group_t**) group);
-    OMPI_ERRHANDLER_RETURN(ret, ompi_win, ret, FUNC_NAME);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
 }

ompi/mpi/c/win_get_name.c

@@ -34,20 +34,19 @@ static const char FUNC_NAME[] = "MPI_Win_get_name";
 
 int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen) 
 {
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
     int ret;
 
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
 
-        if (MPI_WIN_NULL == win || ompi_win_invalid(ompi_win))
+        if (ompi_win_invalid(win)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
-
+        } else if (NULL == win_name || NULL == resultlen) {
-        if (NULL == win_name || NULL == resultlen)
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
     }
 
-    ret = ompi_win_get_name(ompi_win, win_name, resultlen);
+    ret = ompi_win_get_name(win, win_name, resultlen);
 
     OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
 }

ompi/mpi/c/win_lock.c

@@ -49,8 +49,14 @@ int MPI_Win_lock(int lock_type, int rank, int assert, MPI_Win win)
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
         } else if (0 != (assert & ~(MPI_MODE_NOCHECK))) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        } else if (0 != win->w_mode) {
+        } else if (0 != (ompi_win_get_mode(win) &
-            /* window can not be in use at all at this point */
+                         (OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED |
+                          OMPI_WIN_STARTED))) {
+            /* can't be in exposure epoch, or posted, or started */
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
+        } else if ((0 != (ompi_win_get_mode(win) & OMPI_WIN_ACCESS_EPOCH)) &&
+                   (0 == (ompi_win_get_mode(win) & OMPI_WIN_LOCK_ACCESS))) {
+            /* can't be in an access epoch unless we're locked */
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
         } else if (! ompi_win_allow_locks(win)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_SYNC, FUNC_NAME);

ompi/mpi/c/win_post.c

@@ -45,8 +45,8 @@ int MPI_Win_post(MPI_Group group, int assert, MPI_Win win)
         } else if (0 != (assert & ~(MPI_MODE_NOCHECK | MPI_MODE_NOSTORE | 
                                     MPI_MODE_NOPUT))) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        } else if (0 != (win->w_mode & (OMPI_WIN_ACCESS_EPOCH |
+        } else if (0 != (ompi_win_get_mode(win) &
-                                        OMPI_WIN_EXPOSE_EPOCH))) {
+                         (OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_EXPOSE_EPOCH))) {
             /* we can't already be in an an exposure or accesss epoch
                when we start a post */
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);

ompi/mpi/c/win_set_attr.c

@@ -39,9 +39,9 @@ int MPI_Win_set_attr(MPI_Win win, int win_keyval, void *attribute_val)
 
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-	if (MPI_WIN_NULL == win) {
+
-	    return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, 
+        if (ompi_win_invalid(win)) {
-					 FUNC_NAME);
+	    return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
 	}
     }
 

ompi/mpi/c/win_set_errhandler.c

@@ -34,34 +34,26 @@ static const char FUNC_NAME[] = "MPI_Win_set_errhandler";
 
 int MPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler) 
 {
-  if (MPI_PARAM_CHECK) {
+    if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-  }
 
-  if (MPI_PARAM_CHECK) {
+        if (ompi_win_invalid(win)) {
-    if (NULL == win || 
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG, FUNC_NAME);
-        MPI_WIN_NULL == win) {
+        } else if (NULL == errhandler ||
-      return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
+                   MPI_ERRHANDLER_NULL == errhandler ||
-                                   "MPI_Win_set_errhandler");
+                   (OMPI_ERRHANDLER_TYPE_WIN != errhandler->eh_mpi_object_type && 
-    } else if (NULL == errhandler ||
+                    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
-               MPI_ERRHANDLER_NULL == errhandler ||
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-               (OMPI_ERRHANDLER_TYPE_WIN != errhandler->eh_mpi_object_type && 
+        }
-		OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
-      return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG,
-                               "MPI_Win_set_errhandler");
     }
-  }
 
-  /* Ditch the old errhandler, and decrement its refcount */
+    /* Ditch the old errhandler, and decrement its refcount */
+    OBJ_RELEASE(win->error_handler);
 
-  OBJ_RELEASE(win->error_handler);
+    /* We have a valid comm and errhandler, so increment its refcount */
+    win->error_handler = errhandler;
+    OBJ_RETAIN(win->error_handler);
 
-  /* We have a valid comm and errhandler, so increment its refcount */
+    /* All done */
-
+    return MPI_SUCCESS;
-  win->error_handler = errhandler;
-  OBJ_RETAIN(win->error_handler);
-
-  /* All done */
-  
-  return MPI_SUCCESS;
 }

ompi/mpi/c/win_set_name.c

@@ -34,20 +34,19 @@ static const char FUNC_NAME[] = "MPI_Win_set_name";
 
 int MPI_Win_set_name(MPI_Win win, char *win_name) 
 {
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
     int ret;
 
     if (MPI_PARAM_CHECK) {
         OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
 
-        if (MPI_WIN_NULL == win || ompi_win_invalid(ompi_win))
+        if (MPI_WIN_NULL == win || ompi_win_invalid(win)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
-
+        } else if (NULL == win_name) {
-        if (NULL == win_name)
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
     }
 
-    ret = ompi_win_set_name(ompi_win, win_name);
+    ret = ompi_win_set_name(win, win_name);
 
     OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
 }

ompi/mpi/c/win_start.c

@@ -44,8 +44,8 @@ int MPI_Win_start(MPI_Group group, int assert, MPI_Win win)
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
         } else if (0 != (assert & ~(MPI_MODE_NOCHECK))) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        } else if (0 != (win->w_mode & (OMPI_WIN_ACCESS_EPOCH |
+        } else if (0 != (ompi_win_get_mode(win) & 
-                                        OMPI_WIN_EXPOSE_EPOCH))) {
+                         (OMPI_WIN_ACCESS_EPOCH | OMPI_WIN_EXPOSE_EPOCH))) {
             /* we can't already be in an an exposure or accesss epoch
                when we start a start */
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);

ompi/mpi/c/win_test.c

@@ -42,7 +42,7 @@ int MPI_Win_test(MPI_Win win, int *flag)
 
         if (ompi_win_invalid(win)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 == (win->w_mode & OMPI_WIN_POSTED)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_POSTED)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
         }
     }

ompi/mpi/c/win_unlock.c

@@ -44,7 +44,7 @@ int MPI_Win_unlock(int rank, MPI_Win win)
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
         } else if (ompi_win_peer_invalid(win, rank)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
-        } else if (0 == (win->w_mode & OMPI_WIN_LOCK_ACCESS)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_LOCK_ACCESS)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
         }
     }

ompi/mpi/c/win_wait.c

@@ -42,7 +42,7 @@ int MPI_Win_wait(MPI_Win win)
 
         if (ompi_win_invalid(win)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 == (win->w_mode & OMPI_WIN_POSTED)) {
+        } else if (0 == (ompi_win_get_mode(win) & OMPI_WIN_POSTED)) {
             return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_CONFLICT, FUNC_NAME);
         }
     }

ompi/win/win.c

@@ -21,13 +21,20 @@
 #include "mpi.h"
 #include "win/win.h"
 #include "errhandler/errhandler.h"
-#include "include/constants.h"
+#include "ompi/include/constants.h"
 #include "attribute/attribute.h"
 #include "group/group.h"
 #include "info/info.h"
 #include "mca/osc/base/base.h"
 #include "mca/osc/osc.h"
 
+
+/*
+ * Table for Fortran <-> C communicator handle conversion.  Note that
+ * these are not necessarily global.
+ */
+ompi_pointer_array_t ompi_mpi_windows; 
+
 ompi_win_t ompi_mpi_win_null;
 
 static void ompi_win_construct(ompi_win_t *win);
@@ -39,10 +46,15 @@ OBJ_CLASS_INSTANCE(ompi_win_t, opal_object_t,
 int
 ompi_win_init(void)
 {
+    /* setup window Fortran array */
+    OBJ_CONSTRUCT(&ompi_mpi_windows, ompi_pointer_array_t);
+
+    /* Setup MPI_WIN_NULL */
     OBJ_CONSTRUCT(&ompi_mpi_win_null, ompi_win_t);
     ompi_mpi_win_null.w_flags = OMPI_WIN_INVALID;
     ompi_mpi_win_null.w_group = &ompi_mpi_group_null;
     ompi_win_set_name(&ompi_mpi_win_null, "MPI_WIN_NULL");
+    ompi_pointer_array_set_item(&ompi_mpi_windows, 0, &ompi_mpi_win_null);
 
     return OMPI_SUCCESS;
 }
@@ -52,6 +64,7 @@ int
 ompi_win_finalize(void)
 {
     OBJ_DESTRUCT(&ompi_mpi_win_null);
+    OBJ_DESTRUCT(&ompi_mpi_windows);
 
     return OMPI_SUCCESS;
 }
@@ -111,6 +124,13 @@ ompi_win_create(void *base, long size,
         return ret;
     }
 
+    /* fill in Fortran index */
+    win->w_f_to_c_index = ompi_pointer_array_add(&ompi_mpi_windows, win);
+    if (-1 == win->w_f_to_c_index) {
+        ompi_win_free(win);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
     *newwin = win;
 
     return OMPI_SUCCESS;
@@ -122,6 +142,12 @@ ompi_win_free(ompi_win_t *win)
 {
     int ret = win->w_osc_module->osc_free(win);
 
+    if (-1 != win->w_f_to_c_index) {
+        ompi_pointer_array_set_item(&ompi_mpi_windows,
+                                    win->w_f_to_c_index,
+                                    NULL);
+    }
+
     if (OMPI_SUCCESS == ret) {
         OBJ_RELEASE(win);
     }

ompi/win/win.h

@@ -46,6 +46,8 @@ extern "C" {
 #define OMPI_WIN_STARTED      0x00000020
 #define OMPI_WIN_LOCK_ACCESS  0x00000040
 
+OMPI_DECLSPEC extern ompi_pointer_array_t ompi_mpi_windows;
+
 struct ompi_win_t {
     opal_object_t w_base;
 
@@ -53,67 +55,91 @@ struct ompi_win_t {
 
     char w_name[MPI_MAX_OBJECT_NAME];
 
+    /* Group associated with this window. */
     ompi_group_t *w_group;
 
+    /* Information about the state of the window.  */
+    uint16_t w_flags;
+
     /* Attributes */
     opal_hash_table_t *w_keyhash;
 
     /* index in Fortran <-> C translation array */
     int w_f_to_c_index;
 
-    /* Error handling.  This field does not have the "w_" prefix so that
+    /* Error handling.  This field does not have the "w_" prefix so
-       the OMPI_ERRHDL_* macros can find it, regardless of whether it's a
+       that the OMPI_ERRHDL_* macros can find it, regardless of
-       comm, window, or file. */
+       whether it's a comm, window, or file. */
     ompi_errhandler_t                    *error_handler;
     ompi_errhandler_type_t               errhandler_type;
 
     /* displacement factor */
     int w_disp_unit;
 
-    uint16_t w_flags;
-    uint16_t w_mode;
-
     void *w_baseptr;
     long w_size;
 
+    /** Current epoch / mode (access, expose, lock, etc.).  Checked by
+        the argument checking code in the MPI layer, set by the OSC
+        component.  Modified without locking w_lock. */
+    volatile uint16_t w_mode;
+
     /* one sided interface */
     ompi_osc_base_module_t *w_osc_module;
 };
 typedef struct ompi_win_t ompi_win_t;
-
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_win_t);
 
-    int ompi_win_init(void);
+OMPI_DECLSPEC extern ompi_win_t ompi_mpi_win_null;
-    int ompi_win_finalize(void);
 
-    int ompi_win_create(void *base, long size, int disp_unit, 
+int ompi_win_init(void);
-                        ompi_communicator_t *comm, ompi_info_t *info,
+int ompi_win_finalize(void);
-                        ompi_win_t **newwin);
 
-    int ompi_win_free(ompi_win_t *win);
+int ompi_win_create(void *base, long size, int disp_unit, 
+                    ompi_communicator_t *comm, ompi_info_t *info,
+                    ompi_win_t **newwin);
 
-    int ompi_win_set_name(ompi_win_t *win, char *win_name);
+int ompi_win_free(ompi_win_t *win);
-    int ompi_win_get_name(ompi_win_t *win, char *win_name, int *length);
 
-    int ompi_win_group(ompi_win_t *win, ompi_group_t **group);
+int ompi_win_set_name(ompi_win_t *win, char *win_name);
+int ompi_win_get_name(ompi_win_t *win, char *win_name, int *length);
 
-    static inline int ompi_win_invalid(ompi_win_t *win) {
+int ompi_win_group(ompi_win_t *win, ompi_group_t **group);
-        if (NULL == win || (OMPI_WIN_INVALID & win->w_flags)) return true;
+
+static inline int ompi_win_invalid(ompi_win_t *win) {
+    if (NULL == win || 
+        MPI_WIN_NULL == win ||
+        (OMPI_WIN_INVALID & win->w_flags) ||
+        (OMPI_WIN_FREED & win->w_flags)) {
+        return true;
+    } else {
         return false;
     }
+}
 
-    static inline int ompi_win_peer_invalid(ompi_win_t *win, int peer) {
+static inline int ompi_win_peer_invalid(ompi_win_t *win, int peer) {
-        if (win->w_group->grp_proc_count <= peer) return true;
+    if (win->w_group->grp_proc_count <= peer) return true;
-        return false;
+    return false;
-    }
+}
 
-    static inline int ompi_win_rank(ompi_win_t *win) {
+static inline int ompi_win_rank(ompi_win_t *win) {
-        return win->w_group->grp_my_rank;
+    return win->w_group->grp_my_rank;
-    }
+}
 
-    static inline bool ompi_win_allow_locks(ompi_win_t *win) {
+static inline bool ompi_win_allow_locks(ompi_win_t *win) {
-        return (0 != (win->w_flags & OMPI_WIN_NO_LOCKS));
+    return (0 != (win->w_flags & OMPI_WIN_NO_LOCKS));
-    }
+}
+
+static inline int16_t ompi_win_get_mode(ompi_win_t *win) {
+    int16_t mode = win->w_mode;
+    opal_atomic_rmb();
+    return mode;
+}
+
+static inline void ompi_win_set_mode(ompi_win_t *win, int16_t mode) {
+    win->w_mode = mode;
+    opal_atomic_wmb();
+}
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }