diff --git a/ompi/mca/pml/bfo/.opal_ignore b/ompi/mca/pml/bfo/.opal_ignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ompi/mca/pml/bfo/Makefile.am b/ompi/mca/pml/bfo/Makefile.am deleted file mode 100644 index 7565d84c13..0000000000 --- a/ompi/mca/pml/bfo/Makefile.am +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. -# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. -# -# Copyright (c) 2017 IBM Corporation. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -AM_CPPFLAGS = -DPML_BFO=1 - -dist_ompidata_DATA = \ - help-mpi-pml-bfo.txt - -EXTRA_DIST = post_configure.sh - -bfo_sources = \ - pml_bfo.c \ - pml_bfo.h \ - pml_bfo_comm.c \ - pml_bfo_comm.h \ - pml_bfo_component.c \ - pml_bfo_component.h \ - pml_bfo_failover.c \ - pml_bfo_failover.h \ - pml_bfo_hdr.h \ - pml_bfo_iprobe.c \ - pml_bfo_irecv.c \ - pml_bfo_isend.c \ - pml_bfo_progress.c \ - pml_bfo_rdma.c \ - pml_bfo_rdma.h \ - pml_bfo_rdmafrag.c \ - pml_bfo_rdmafrag.h \ - pml_bfo_recvfrag.c \ - pml_bfo_recvfrag.h \ - pml_bfo_recvreq.c \ - pml_bfo_recvreq.h \ - pml_bfo_sendreq.c \ - pml_bfo_sendreq.h \ - pml_bfo_start.c - -# If we have CUDA support requested, build the CUDA file also -if OPAL_cuda_support -bfo_sources += \ - pml_bfo_cuda.c -endif - -if MCA_BUILD_ompi_pml_bfo_DSO -component_noinst = -component_install = mca_pml_bfo.la -else -component_noinst = libmca_pml_bfo.la -component_install = -endif - -mcacomponentdir = $(ompilibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_pml_bfo_la_SOURCES = $(bfo_sources) -mca_pml_bfo_la_LDFLAGS = -module -avoid-version -mca_pml_bfo_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la - -noinst_LTLIBRARIES = $(component_noinst) -libmca_pml_bfo_la_SOURCES = $(bfo_sources) -libmca_pml_bfo_la_LDFLAGS = -module -avoid-version diff --git a/ompi/mca/pml/bfo/README b/ompi/mca/pml/bfo/README deleted file mode 100644 index 88c3b1a70f..0000000000 --- a/ompi/mca/pml/bfo/README +++ /dev/null @@ -1,340 +0,0 @@ -Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - -BFO DESIGN DOCUMENT -This document describes the use and design of the bfo. In addition, -there is a section at the end explaining why this functionality was -not merged into the ob1 PML. - -1. GENERAL USAGE -First, one has to configure the failover code into the openib BTL so -that bfo will work correctly. To do this: -configure --enable-btl-openib-failover. - -Then, when running one needs to select the bfo PML explicitly. -mpirun --mca pml bfo - -Note that one needs to both configure with --enable-btl-openib-failover -and run with --mca pml bfo to get the failover support. If one of -these two steps is skipped, then the MPI job will just abort in the -case of an error like it normally does with the ob1 PML. - -2. GENERAL FUNCTION -The bfo failover feature requires two or more openib BTLs in use. In -normal operation, it will stripe the communication over the multiple -BTLs. When an error is detected, it will stop using the BTL that -incurred the error and continue the communication over the remaining -BTL. Once a BTL has been mapped out, it cannot be used by the job -again, even if the underlying fabric becomes functional again. Only -new jobs started after the fabric comes back up will use both BTLs. - -The bfo works in conjunction with changes that were made in the openib -BTL. As noted above, those changes need to be configured into the -BTL for everything to work properly. - -The bfo only fails over between openib BTLs. It cannot failover from -an openib BTL to TCP, for example. - -3. GENERAL DESIGN -The bfo (Btl FailOver) PML was designed to work in clusters that have -multiple openib BTLs. It was designed to be lightweight so as to -avoid any adverse effects on latency. To that end, there is no -tracking of fragments or messages in the bfo PML. Rather, it depends -on the underlying BTL to notify it of each fragment that has an error. -The bfo then decides what needs to be done based on the type of -fragment that gets an error. - -No additional sequence numbers were introduced in the bfo. Instead, -it makes use of the sequence numbers that exist in the MATCH, RNDV and -RGET fragment header. In that way, duplicate fragments that have -MATCH information in them can be detected. Other fragments, like PUT -and ACK, are never retransmitted so it does not matter that they do -not have sequence numbers. The FIN header was a special case in that -it was changed to include the MATCH header so that the tag, source, -and context fields could be used to check for duplicate FINs. - -Note that the assumption is that the underlying BTL will always issue -a callback with an error flag when it thinks a fragment has an error. -This means that even after an error is detected on a BTL, the BTL -continues to be checked for any other messages that may also complete -with an error. This is potentially a unique characteristic of the -openib BTL when running over RC connections that allows the BFO to -work properly. - -One scenario that is particularly difficult to handle is the case -where a fragment has an error but the message actually makes it to the -other side. It is because of this that all fragments need to be -checked to make sure they are not a duplicate. This scenario also -complicates some of the rendezvous protocols as the two sides may not -agree where the problem occurred. For example, one can imagine a -sender getting an error on a final FIN message, but the FIN message -actually arrives at the other side. The receiver thinks the -communication is done and moves on. The sender thinks there was a -problem, and that the communication needs to restart. - -It is also important to note that a message cannot signal a successful -completion and *not* make it to the receiver. This would probably cause -the bfo to hang. - -4. ERRORS -Errors are detected in the openib BTL layer and propagated to the PML -layer. Typically, the errors occur while polling the completion -queue, but can happen in other areas as well. When an error occurs, -an additional callback is called so the PML can map out the connection -for future sending. Then the callback associated with the fragment is -called, but with the error field set to OMPI_ERROR. This way, the PML -knows that this fragment may not have made it to the remote side. - -The first callback into the PML is via the mca_pml_bfo_error_handler() -callback and the PML uses this to remove a connection for future -sending. If the error_proc_t field is NULL, then the entire BTL is -removed for any future communication. If the error_proc_t is not -NULL, then the BTL is only removed for the connection associated with -the error_proc_t. - -The second callback is the standard one for a completion event, and -this can trigger various activities in the PML. The regular callback -function is called but the status is set to OMPI_ERROR. The PML layer -detects this and calls some failover specific routines depending on -the type of fragment that got the error. - - -5. RECOVERY OF MATCH FRAGMENTS -Note: For a general description of how the various fragments interact, -see Appendix 1 at the end of this document. - -In the case of a MATCH fragment, the fragment is simply resent. Care -has to be taken with a MATCH fragment that is sent via the standard -interface and one that is sent via the sendi interface. In the -standard send, the send request is still available and is therefore -reset reused to send the MATCH fragment. In the case of the sendi -fragment, the send request is gone, so the fragment is regenerated -from the information contained within the fragment. - -6. RECOVERY OF RNDV or LARGE MESSAGE RDMA -In the case of a large message RDMA transfer or a RNDV transfer where -the message consists of several fragments, the restart is a little -more complicated. This includes fragments like RNDV, PUT, RGET, FRAG, -FIN, and RDMA write and RDMA read completions. In most cases, the -requests associated with these fragments are reset and restarted. - -First, it should be pointed out that a new variable was added to the -send and receive requests. This variable tracks outstanding send -events that have not yet received their completion events. This new -variable is used so that a request is not restarted until all the -outstanding events have completed. If one does not wait for the -outstanding events to complete, then one may restart a request and -then a completion event will happen on the wrong request. - -There is a second variable added to each request and that is one that -shows whether the request is already in an error state. When a request -reaches the state that it has an error flagged on it and the outstanding -completion events are down to zero, it can start the restart dance -as described below. - -7. SPECIAL CASE FOR FIN FRAGMENT -Like the MATCH fragment, the FIN message is also simply resent. Like -the sendi MATCH fragment, there may be no request associated with the -FIN message when it gets an error, so the fragment is recreated from -the information in the fragment. The FIN fragment was modified to -have additional information like what is in a MATCH fragment including -the context, source, and tag. In this way, we can figure out if the -FIN message is a duplicate on the receiving side. - -8. RESTART DANCE -When the bfo determines that there are no outstanding completion events, -a restart dance is initiated. There are four new PML message types that -have been created to participate in the dance. - 1. RNDVRESTARTNOTIFY - 2. RECVERRNOTIFY - 3. RNDVRESTARTACK - 4. RNDVRESTARTNACK - -When the send request is in an error state and the outstanding -completion events is zero, RNDVRESTARTNOTIFY is sent from the sender -to the receiver to let it know that the communication needs to be -restarted. Upon receipt of the RNDVRESTARTNOTIFY, the receiver first -checks to make sure that it is still pointing to a valid receiver -request. If so, it marks the receive request in error. It then -checks to see if there are any outstanding completion events on the -receiver. If there are no outstanding completion events, the receiver -sends the RNDVRESTARTACK. If there are outstanding completion events, -then the RNDVRESTARTACK gets sent later when a completion event occurs -that brings the outstanding event count to zero. - -In the case that the receiver determines that it is no longer looking -at a valid receive request, which means the request is complete, the -receiver responds with a RNDVRESTARTNACK. While rare, this case can -happen for example, when a final FRAG message triggers an error on the -sender, but actually makes it to the receiver. - -The RECVERRNOTIFY fragment is used so the receiver can let the sender -sender know that it had an error. The sender then waits for all of -its completion events, and then sends a RNDVRESTARTNOTIFY. - -All the handling of these new messages is contained in the -pml_bfo_failover files. - -9. BTL SUPPORT -The openib BTL also supplies a lot of support for the bfo PML. First, -fragments can be stored in the BTL during normal operation if -resources become scarce. This means that when an error is detected in -the BTL, it needs to scour its internal queues for fragments that are -destined for the BTL and error them out. The function -error_out_all_pending_frags() takes care of this functionality. And -some of the fragments stored can be coalesced, so care has to be taken -to tease out each message from a coalesced fragment. - -There is also some special code in the BTL to handle some strange -occurrences that were observed in the BTL. First, there are times -where only one half of the connection gets an error. This can result -in a mismatch between what the PML thinks is available to it and can -cause hangs. Therefore, when a BTL detects an error, it sends a -special message down the working BTL connection to tell the remote -side that it needs to be brought down as well. - -Secondly, it has been observed that a message can get stuck in the -eager RDMA connection between two BTLs. In this case, an error is -detected on one side, but the other side never sees the message. -Therefore, a special message is sent to the other side telling it to -move along in the eager RDMA connection. This is all somewhat -confusing. See the code in the btl_openib_failover.c file for the -details. - -10. MERGING -Every effort was made to try and merge the bfo PML into the ob1 PML. -The idea was that any upgrades to the ob1 PML would automatically make -it into the bfo PML and this would enhance maintainability of all the -code. However, it was deemed that this merging would cause more -problems than it would solve. What was attempted and why the -conclusion was made are documented here. - -One can look at the bfo and easily see the differences between it and -ob1. All the bfo specific code is surrounded by #if PML_BFO. In -addition, there are two additional files in the bfo, -pml_bfo_failover.c and pml_bfo_failover.h. - -To merge them, the following was attempted. First, add all the code -in #if regions into the ob1 PML. As of this writing, there are 73 -#ifs that would have to be added into ob1. - -Secondly, remove almost all the pml_bfo files and replace them with -links to the ob1 files. - -Third, create a new header file that did name shifting of all the -functions so that ob1 and bfo could live together. This also included -having to create macros for the names of header files as well. To -help illustrate the name shifting issue, here is what the file might -look like in the bfo directory. - -/* Need macros for the header files as they are different in the - * different PMLs */ -#define PML "bfo" -#define PML_OB1_H "pml_bfo.h" -#define PML_OB1_COMM_H "pml_bfo_comm.h" -#define PML_OB1_COMPONENT_H "pml_bfo_component.h" -#define PML_OB1_HDR_H "pml_bfo_hdr.h" -#define PML_OB1_RDMA_H "pml_bfo_rdma.h" -#define PML_OB1_RDMAFRAG_H "pml_bfo_rdmafrag.h" -#define PML_OB1_RECVFRAG_H "pml_bfo_recvfrag.h" -#define PML_OB1_RECVREQ_H "pml_bfo_recvreq.h" -#define PML_OB1_SENDREQ_H "pml_bfo_sendreq.h" - -/* Name shifting of functions from ob1 to bfo (incomplete list) */ -#define mca_pml_ob1 mca_pml_bfo -#define mca_pml_ob1_t mca_pml_bfo_t -#define mca_pml_ob1_component mca_pml_bfo_component -#define mca_pml_ob1_add_procs mca_pml_bfo_add_procs -#define mca_pml_ob1_del_procs mca_pml_bfo_del_procs -#define mca_pml_ob1_enable mca_pml_bfo_enable -#define mca_pml_ob1_progress mca_pml_bfo_progress -#define mca_pml_ob1_add_comm mca_pml_bfo_add_comm -#define mca_pml_ob1_del_comm mca_pml_bfo_del_comm -#define mca_pml_ob1_irecv_init mca_pml_bfo_irecv_init -#define mca_pml_ob1_irecv mca_pml_bfo_irecv -#define mca_pml_ob1_recv mca_pml_bfo_recv -#define mca_pml_ob1_isend_init mca_pml_bfo_isend_init -#define mca_pml_ob1_isend mca_pml_bfo_isend -#define mca_pml_ob1_send mca_pml_bfo_send -#define mca_pml_ob1_iprobe mca_pml_bfo_iprobe -[...and much more ...] - -The pml_bfo_hdr.h file was not a link because the changes in it were -so extensive. Also the Makefile was kept separate so it could include -the additional failover files as well as add a compile directive that -would force the files to be compiled as bfo instead of ob1. - -After these changes were made, several independent developers reviewed -the results and concluded that making these changes would have too -much of a negative impact on ob1 maintenance. First, the code became -much harder to read with all the additional #ifdefs. Secondly, the -possibility of adding other features, like csum, to ob1 would only -make this issue even worse. Therefore, it was decided to keep the bfo -PML separate from ob1. - -11. UTILITIES -In an ideal world, any bug fixes that are made in the ob1 PML would -also be made in the csum and the bfo PMLs. However, that does not -always happen. Therefore, there are two new utilities added to the -contrib directory. - -check-ob1-revision.pl -check-ob1-pml-diffs.pl - -The first one can be run to see if ob1 has changed from its last known -state. Here is an example. - - machine =>check-ob1-revision.pl -Running svn diff -r24138 ../ompi/mca/pml/ob1 -No new changes detected in ob1. Everything is fine. - -If there are differences, then one needs to review them and potentially -add them to the bfo (and csum also if one feels like it). -After that, bump up the value in the script to the latest value. - -The second script allows one to see the differences between the ob1 -and bfo PML. Here is an example. - - machine =>check-ob1-pml-diffs.pl - -Starting script to check differences between bfo and ob1... -Files Compared: pml_ob1.c and pml_bfo.c -No differences encountered -Files Compared: pml_ob1.h and pml_bfo.h -[...snip...] -Files Compared: pml_ob1_start.c and pml_bfo_start.c -No differences encountered - -There is a lot more in the script that tells how it is used. - - -Appendix 1: SIMPLE OVERVIEW OF COMMUNICATION PROTOCOLS -The drawings below attempt to describe some of the general flow of -fragments in the various protocols that are supported in the PMLs. -The "read" and "write" are actual RDMA actions and do not pertain to -fragments that are sent. As can be inferred, they use FIN messages to -indicate their completion. - - -MATCH PROTOCOL -sender >->->-> MATCH >->->-> receiver - -SEND WITH MULTIPLE FRAGMENTS -sender >->->-> RNDV >->->-> receiver - <-<-<-< ACK <-<-<-< - >->->-> FRAG >->->-> - >->->-> FRAG >->->-> - >->->-> FRAG >->->-> - -RDMA PUT -sender >->->-> RNDV >->->-> receiver - <-<-<-< PUT <-<-<-< - <-<-<-< PUT <-<-<-< - >->->-> write >->->-> - >->->-> FIN >->->-> - >->->-> write >->->-> - >->->-> FIN >->->-> - -RMA GET -sender >->->-> RGET >->->-> receiver - <-<-<-< read <-<-<-< - <-<-<-< FIN <-<-<-< diff --git a/ompi/mca/pml/bfo/configure.m4 b/ompi/mca/pml/bfo/configure.m4 deleted file mode 100644 index 4001c94d65..0000000000 --- a/ompi/mca/pml/bfo/configure.m4 +++ /dev/null @@ -1,27 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2013 Sandia National Laboratories. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_ompi_pml_bfo_POST_CONFIG(will_build) -# ---------------------------------------- -# The BFO PML requires a BML endpoint tag to compile, so require it. -# Require in POST_CONFIG instead of CONFIG so that we only require it -# if we're not disabled. -AC_DEFUN([MCA_ompi_pml_bfo_POST_CONFIG], [ - AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])]) -])dnl - -# MCA_ompi_pml_bfo_CONFIG(action-if-can-compile, -# [action-if-cant-compile]) -# ------------------------------------------------ -# We can always build, unless we were explicitly disabled. -AC_DEFUN([MCA_ompi_pml_bfo_CONFIG],[ - AC_CONFIG_FILES([ompi/mca/pml/bfo/Makefile]) - [$1] -])dnl diff --git a/ompi/mca/pml/bfo/help-mpi-pml-bfo.txt b/ompi/mca/pml/bfo/help-mpi-pml-bfo.txt deleted file mode 100644 index 4bbff8ff6d..0000000000 --- a/ompi/mca/pml/bfo/help-mpi-pml-bfo.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -*- text -*- -# -# Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -[eager_limit_too_small] -The "eager limit" MCA parameter in the %s BTL was set to a value which -is too low for Open MPI to function properly. Please re-run your job -with a higher eager limit value for this BTL; the exact MCA parameter -name and its corresponding minimum value is shown below. - - Local host: %s - BTL name: %s - BTL eager limit value: %d (set via btl_%s_eager_limit) - BTL eager limit minimum: %d - MCA parameter name: btl_%s_eager_limit diff --git a/ompi/mca/pml/bfo/owner.txt b/ompi/mca/pml/bfo/owner.txt deleted file mode 100644 index f1dfe8edb4..0000000000 --- a/ompi/mca/pml/bfo/owner.txt +++ /dev/null @@ -1,7 +0,0 @@ -# -# owner/status file -# owner: institution that is responsible for this package -# status: e.g. active, maintenance, unmaintained -# -owner: NVIDIA -status: unmaintained diff --git a/ompi/mca/pml/bfo/pml_bfo.c b/ompi/mca/pml/bfo/pml_bfo.c deleted file mode 100644 index ce33b0d57b..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo.c +++ /dev/null @@ -1,897 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; -*- */ -/* - * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2006-2008 University of Houston. All rights reserved. - * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2011-2012 Los Alamos National Security, LLC. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include -#include - -#include "opal/class/opal_bitmap.h" -#include "opal/util/output.h" -#include "opal/util/show_help.h" -#include "opal/mca/btl/btl.h" -#include "opal/mca/btl/base/base.h" -#include "opal/mca/pmix/pmix.h" - -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/pml/base/base.h" -#include "ompi/mca/pml/base/base.h" -#include "ompi/mca/bml/base/base.h" -#include "ompi/runtime/ompi_cr.h" - -#include "pml_bfo.h" -#include "pml_bfo_component.h" -#include "pml_bfo_comm.h" -#include "pml_bfo_hdr.h" -#include "pml_bfo_recvfrag.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_rdmafrag.h" -#if PML_BFO -#include "pml_bfo_failover.h" -#endif /* PML_BFO */ - -mca_pml_bfo_t mca_pml_bfo = { - { - mca_pml_bfo_add_procs, - mca_pml_bfo_del_procs, - mca_pml_bfo_enable, - mca_pml_bfo_progress, - mca_pml_bfo_add_comm, - mca_pml_bfo_del_comm, - mca_pml_bfo_irecv_init, - mca_pml_bfo_irecv, - mca_pml_bfo_recv, - mca_pml_bfo_isend_init, - mca_pml_bfo_isend, - mca_pml_bfo_send, - mca_pml_bfo_iprobe, - mca_pml_bfo_probe, - mca_pml_bfo_start, - mca_pml_bfo_improbe, - mca_pml_bfo_mprobe, - mca_pml_bfo_imrecv, - mca_pml_bfo_mrecv, - mca_pml_bfo_dump, - mca_pml_bfo_ft_event, - 65535, - INT_MAX - } -}; - - -void mca_pml_bfo_error_handler( struct mca_btl_base_module_t* btl, - int32_t flags, ompi_proc_t* errproc, - char* btlinfo ); - -int mca_pml_bfo_enable(bool enable) -{ - if( false == enable ) { - return OMPI_SUCCESS; - } - - OBJ_CONSTRUCT(&mca_pml_bfo.lock, opal_mutex_t); - - /* fragments */ - OBJ_CONSTRUCT(&mca_pml_bfo.rdma_frags, opal_free_list_t); - opal_free_list_init( &mca_pml_bfo.rdma_frags, - sizeof(mca_pml_bfo_rdma_frag_t), - opal_cache_line_size, - OBJ_CLASS(mca_pml_bfo_rdma_frag_t), - 0,opal_cache_line_size, - mca_pml_bfo.free_list_num, - mca_pml_bfo.free_list_max, - mca_pml_bfo.free_list_inc, - NULL, 0, NULL, NULL, NULL ); - - OBJ_CONSTRUCT(&mca_pml_bfo.recv_frags, opal_free_list_t); - opal_free_list_init( &mca_pml_bfo.recv_frags, - sizeof(mca_pml_bfo_recv_frag_t) + mca_pml_bfo.unexpected_limit, - opal_cache_line_size, - OBJ_CLASS(mca_pml_bfo_recv_frag_t), - 0,opal_cache_line_size, - mca_pml_bfo.free_list_num, - mca_pml_bfo.free_list_max, - mca_pml_bfo.free_list_inc, - NULL, 0, NULL, NULL, NULL ); - - OBJ_CONSTRUCT(&mca_pml_bfo.pending_pckts, opal_free_list_t); - opal_free_list_init( &mca_pml_bfo.pending_pckts, - sizeof(mca_pml_bfo_pckt_pending_t), - opal_cache_line_size, - OBJ_CLASS(mca_pml_bfo_pckt_pending_t), - 0,opal_cache_line_size, - mca_pml_bfo.free_list_num, - mca_pml_bfo.free_list_max, - mca_pml_bfo.free_list_inc, - NULL, 0, NULL, NULL, NULL ); - - OBJ_CONSTRUCT(&mca_pml_bfo.buffers, opal_free_list_t); - OBJ_CONSTRUCT(&mca_pml_bfo.send_ranges, opal_free_list_t); - opal_free_list_init( &mca_pml_bfo.send_ranges, - sizeof(mca_pml_bfo_send_range_t) + - (mca_pml_bfo.max_send_per_range - 1) * sizeof(mca_pml_bfo_com_btl_t), - opal_cache_line_size, - OBJ_CLASS(mca_pml_bfo_send_range_t), - 0,opal_cache_line_size, - mca_pml_bfo.free_list_num, - mca_pml_bfo.free_list_max, - mca_pml_bfo.free_list_inc, - NULL, 0, NULL, NULL, NULL ); - - /* pending operations */ - OBJ_CONSTRUCT(&mca_pml_bfo.send_pending, opal_list_t); - OBJ_CONSTRUCT(&mca_pml_bfo.recv_pending, opal_list_t); - OBJ_CONSTRUCT(&mca_pml_bfo.pckt_pending, opal_list_t); - OBJ_CONSTRUCT(&mca_pml_bfo.rdma_pending, opal_list_t); - /* missing communicator pending list */ - OBJ_CONSTRUCT(&mca_pml_bfo.non_existing_communicator_pending, opal_list_t); - - /** - * If we get here this is the PML who get selected for the run. We - * should get ownership for the send and receive requests list, and - * initialize them with the size of our own requests. - */ - opal_free_list_init( &mca_pml_base_send_requests, - sizeof(mca_pml_bfo_send_request_t) + - (mca_pml_bfo.max_rdma_per_request - 1) * - sizeof(mca_pml_bfo_com_btl_t), - opal_cache_line_size, - OBJ_CLASS(mca_pml_bfo_send_request_t), - 0,opal_cache_line_size, - mca_pml_bfo.free_list_num, - mca_pml_bfo.free_list_max, - mca_pml_bfo.free_list_inc, - NULL, 0, NULL, NULL, NULL ); - - opal_free_list_init( &mca_pml_base_recv_requests, - sizeof(mca_pml_bfo_recv_request_t) + - (mca_pml_bfo.max_rdma_per_request - 1) * - sizeof(mca_pml_bfo_com_btl_t), - opal_cache_line_size, - OBJ_CLASS(mca_pml_bfo_recv_request_t), - 0,opal_cache_line_size, - mca_pml_bfo.free_list_num, - mca_pml_bfo.free_list_max, - mca_pml_bfo.free_list_inc, - NULL, 0, NULL, NULL, NULL ); - - mca_pml_bfo.enabled = true; - return OMPI_SUCCESS; -} - -int mca_pml_bfo_add_comm(ompi_communicator_t* comm) -{ - /* allocate pml specific comm data */ - mca_pml_bfo_comm_t* pml_comm = OBJ_NEW(mca_pml_bfo_comm_t); - opal_list_item_t *item, *next_item; - mca_pml_bfo_recv_frag_t* frag; - mca_pml_bfo_comm_proc_t* pml_proc; - mca_pml_bfo_match_hdr_t* hdr; - int i; - - if (NULL == pml_comm) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* should never happen, but it was, so check */ - if (comm->c_contextid > mca_pml_bfo.super.pml_max_contextid) { - OBJ_RELEASE(pml_comm); - return OMPI_ERR_OUT_OF_RESOURCE; - } - - mca_pml_bfo_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count); - comm->c_pml_comm = pml_comm; - - for( i = 0; i < comm->c_remote_group->grp_proc_count; i++ ) { - pml_comm->procs[i].ompi_proc = ompi_group_peer_lookup(comm->c_remote_group,i); - OBJ_RETAIN(pml_comm->procs[i].ompi_proc); - } - /* Grab all related messages from the non_existing_communicator pending queue */ - for( item = opal_list_get_first(&mca_pml_bfo.non_existing_communicator_pending); - item != opal_list_get_end(&mca_pml_bfo.non_existing_communicator_pending); - item = next_item ) { - frag = (mca_pml_bfo_recv_frag_t*)item; - next_item = opal_list_get_next(item); - hdr = &frag->hdr.hdr_match; - - /* Is this fragment for the current communicator ? */ - if( frag->hdr.hdr_match.hdr_ctx != comm->c_contextid ) - continue; - - /* As we now know we work on a fragment for this communicator - * we should remove it from the - * non_existing_communicator_pending list. */ - opal_list_remove_item( &mca_pml_bfo.non_existing_communicator_pending, - item ); - - add_fragment_to_unexpected: - - /* We generate the MSG_ARRIVED event as soon as the PML is aware - * of a matching fragment arrival. Independing if it is received - * on the correct order or not. This will allow the tools to - * figure out if the messages are not received in the correct - * order (if multiple network interfaces). - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - /* There is no matching to be done, and no lock to be held on the communicator as - * we know at this point that the communicator has not yet been returned to the user. - * The only required protection is around the non_existing_communicator_pending queue. - * We just have to push the fragment into the unexpected list of the corresponding - * proc, or into the out-of-order (cant_match) list. - */ - pml_proc = &(pml_comm->procs[hdr->hdr_src]); - - if( ((uint16_t)hdr->hdr_seq) == ((uint16_t)pml_proc->expected_sequence) ) { - /* We're now expecting the next sequence number. */ - pml_proc->expected_sequence++; - opal_list_append( &pml_proc->unexpected_frags, (opal_list_item_t*)frag ); - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - /* And now the ugly part. As some fragments can be inserted in the cant_match list, - * every time we succesfully add a fragment in the unexpected list we have to make - * sure the next one is not in the cant_match. Otherwise, we will endup in a deadlock - * situation as the cant_match is only checked when a new fragment is received from - * the network. - */ - for(frag = (mca_pml_bfo_recv_frag_t *)opal_list_get_first(&pml_proc->frags_cant_match); - frag != (mca_pml_bfo_recv_frag_t *)opal_list_get_end(&pml_proc->frags_cant_match); - frag = (mca_pml_bfo_recv_frag_t *)opal_list_get_next(frag)) { - hdr = &frag->hdr.hdr_match; - /* If the message has the next expected seq from that proc... */ - if(hdr->hdr_seq != pml_proc->expected_sequence) - continue; - - opal_list_remove_item(&pml_proc->frags_cant_match, (opal_list_item_t*)frag); - goto add_fragment_to_unexpected; - } - } else { - opal_list_append( &pml_proc->frags_cant_match, (opal_list_item_t*)frag ); - } - } - return OMPI_SUCCESS; -} - -int mca_pml_bfo_del_comm(ompi_communicator_t* comm) -{ - mca_pml_bfo_comm_t* pml_comm = comm->c_pml_comm; - int i; - - for( i = 0; i < comm->c_remote_group->grp_proc_count; i++ ) { - OBJ_RELEASE(pml_comm->procs[i].ompi_proc); - } - OBJ_RELEASE(comm->c_pml_comm); - comm->c_pml_comm = NULL; - return OMPI_SUCCESS; -} - - -/* - * For each proc setup a datastructure that indicates the BTLs - * that can be used to reach the destination. - * - */ - -int mca_pml_bfo_add_procs(ompi_proc_t** procs, size_t nprocs) -{ - opal_bitmap_t reachable; - int rc; - opal_list_item_t *item; - - if(nprocs == 0) - return OMPI_SUCCESS; - - OBJ_CONSTRUCT(&reachable, opal_bitmap_t); - rc = opal_bitmap_init(&reachable, (int)nprocs); - if(OMPI_SUCCESS != rc) - return rc; - - /* - * JJH: Disable this in FT enabled builds since - * we use a wrapper PML. It will cause this check to - * return failure as all processes will return the wrapper PML - * component in use instead of the wrapped PML component underneath. - */ -#if OPAL_ENABLE_FT_CR == 0 - /* make sure remote procs are using the same PML as us */ - if (OMPI_SUCCESS != (rc = mca_pml_base_pml_check_selected("bfo", - procs, - nprocs))) { - return rc; - } -#endif - - rc = mca_bml.bml_add_procs( nprocs, - procs, - &reachable ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - /* Check that values supplied by all initialized btls will work - for us. Note that this is the list of all initialized BTLs, - not the ones used for the just added procs. This is a little - overkill and inaccurate, as we may end up not using the BTL in - question and all add_procs calls after the first one are - duplicating an already completed check. But the final - initialization of the PML occurs before the final - initialization of the BTLs, and iterating through the in-use - BTLs requires iterating over the procs, as the BML does not - expose all currently in use btls. */ - - for (item = opal_list_get_first(&mca_btl_base_modules_initialized) ; - item != opal_list_get_end(&mca_btl_base_modules_initialized) ; - item = opal_list_get_next(item)) { - mca_btl_base_selected_module_t *sm = - (mca_btl_base_selected_module_t*) item; - if (sm->btl_module->btl_eager_limit < sizeof(mca_pml_bfo_hdr_t)) { - opal_show_help("help-mpi-pml-bfo.txt", "eager_limit_too_small", - true, - sm->btl_component->btl_version.mca_component_name, - ompi_process_info.nodename, - sm->btl_component->btl_version.mca_component_name, - sm->btl_module->btl_eager_limit, - sm->btl_component->btl_version.mca_component_name, - sizeof(mca_pml_bfo_hdr_t), - sm->btl_component->btl_version.mca_component_name); - rc = OMPI_ERR_BAD_PARAM; - goto cleanup_and_return; - } - } - - - /* TODO: Move these callback registration to another place */ - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_MATCH, - mca_pml_bfo_recv_frag_callback_match, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_RNDV, - mca_pml_bfo_recv_frag_callback_rndv, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_RGET, - mca_pml_bfo_recv_frag_callback_rget, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_ACK, - mca_pml_bfo_recv_frag_callback_ack, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_FRAG, - mca_pml_bfo_recv_frag_callback_frag, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_PUT, - mca_pml_bfo_recv_frag_callback_put, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_FIN, - mca_pml_bfo_recv_frag_callback_fin, - NULL ); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - -#if PML_BFO - rc = mca_pml_bfo_register_callbacks(); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; -#endif /* PML_BFO */ - /* register error handlers */ - rc = mca_bml.bml_register_error((mca_btl_base_module_error_cb_fn_t)mca_pml_bfo_error_handler); - if(OMPI_SUCCESS != rc) - goto cleanup_and_return; - - cleanup_and_return: - OBJ_DESTRUCT(&reachable); - - return rc; -} - -/* - * iterate through each proc and notify any PTLs associated - * with the proc that it is/has gone away - */ - -int mca_pml_bfo_del_procs(ompi_proc_t** procs, size_t nprocs) -{ - return mca_bml.bml_del_procs(nprocs, procs); -} - -/* - * diagnostics - */ - -int mca_pml_bfo_dump(struct ompi_communicator_t* comm, int verbose) -{ - struct mca_pml_comm_t* pml_comm = comm->c_pml_comm; - int i; - - /* iterate through all procs on communicator */ - for( i = 0; i < (int)pml_comm->num_procs; i++ ) { - mca_pml_bfo_comm_proc_t* proc = &pml_comm->procs[i]; - mca_bml_base_endpoint_t* ep = (mca_bml_base_endpoint_t*)proc->ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - size_t n; - - opal_output(0, "[Rank %d]\n", i); - /* dump all receive queues */ - - /* dump all btls */ - for(n=0; nbtl_eager.arr_size; n++) { - mca_bml_base_btl_t* bml_btl = &ep->btl_eager.bml_btls[n]; - bml_btl->btl->btl_dump(bml_btl->btl, bml_btl->btl_endpoint, verbose); - } - } - return OMPI_SUCCESS; -} - -static void mca_pml_bfo_fin_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context; - -#if PML_BFO - if( OPAL_UNLIKELY(OMPI_SUCCESS != status) ) { - mca_pml_bfo_repost_fin(des); - return; - } - MCA_PML_BFO_CHECK_EAGER_BML_BTL_ON_FIN_COMPLETION(bml_btl, btl, des); -#endif /* PML_BFO */ - /* check for pending requests */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -/** - * Send an FIN to the peer. If we fail to send this ack (no more available - * fragments or the send failed) this function automatically add the FIN - * to the list of pending FIN, Which guarantee that the FIN will be sent - * later. - */ -int mca_pml_bfo_send_fin( ompi_proc_t* proc, - mca_bml_base_btl_t* bml_btl, - opal_ptr_t hdr_des, - uint8_t order, -#if PML_BFO - uint32_t status, - uint16_t seq, - uint8_t restartseq, - uint16_t ctx, uint32_t src) -#else /* PML_BFO */ - uint32_t status ) -#endif /* PML_BFO */ -{ - mca_btl_base_descriptor_t* fin; - mca_pml_bfo_fin_hdr_t* hdr; - int rc; - - mca_bml_base_alloc(bml_btl, &fin, order, sizeof(mca_pml_bfo_fin_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - - if(NULL == fin) { - MCA_PML_BFO_ADD_FIN_TO_PENDING(proc, hdr_des, bml_btl, order, status); - return OMPI_ERR_OUT_OF_RESOURCE; - } - fin->des_cbfunc = mca_pml_bfo_fin_completion; - fin->des_cbdata = NULL; - - /* fill in header */ - hdr = (mca_pml_bfo_fin_hdr_t*)fin->des_local->seg_addr.pval; - hdr->hdr_common.hdr_flags = 0; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_FIN; - hdr->hdr_des = hdr_des; - hdr->hdr_fail = status; -#if PML_BFO - fin->des_cbdata = proc; - hdr->hdr_match.hdr_seq = seq; - hdr->hdr_match.hdr_ctx = ctx; - hdr->hdr_match.hdr_src = src; - hdr->hdr_match.hdr_common.hdr_flags = restartseq; /* use unused hdr_flags field */ -#endif /* PML_BFO */ - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_FIN, proc); - - /* queue request */ - rc = mca_bml_base_send( bml_btl, - fin, - MCA_PML_BFO_HDR_TYPE_FIN ); - if( OPAL_LIKELY( rc >= 0 ) ) { - if( OPAL_LIKELY( 1 == rc ) ) { - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); - } - return OMPI_SUCCESS; - } - mca_bml_base_free(bml_btl, fin); - MCA_PML_BFO_ADD_FIN_TO_PENDING(proc, hdr_des, bml_btl, order, status); - return OMPI_ERR_OUT_OF_RESOURCE; -} - -void mca_pml_bfo_process_pending_packets(mca_bml_base_btl_t* bml_btl) -{ - mca_pml_bfo_pckt_pending_t *pckt; - int32_t i, rc, s = (int32_t)opal_list_get_size(&mca_pml_bfo.pckt_pending); - - for(i = 0; i < s; i++) { - mca_bml_base_btl_t *send_dst = NULL; - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - pckt = (mca_pml_bfo_pckt_pending_t*) - opal_list_remove_first(&mca_pml_bfo.pckt_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - if(NULL == pckt) - break; - if(pckt->bml_btl != NULL && - pckt->bml_btl->btl == bml_btl->btl) { - send_dst = pckt->bml_btl; - } else { - mca_bml_base_endpoint_t* endpoint = - (mca_bml_base_endpoint_t*) pckt->proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - send_dst = mca_bml_base_btl_array_find( - &endpoint->btl_eager, bml_btl->btl); - } - if(NULL == send_dst) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.pckt_pending, - (opal_list_item_t*)pckt); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - continue; - } - - switch(pckt->hdr.hdr_common.hdr_type) { - case MCA_PML_BFO_HDR_TYPE_ACK: - rc = mca_pml_bfo_recv_request_ack_send_btl(pckt->proc, - send_dst, - pckt->hdr.hdr_ack.hdr_src_req.lval, - pckt->hdr.hdr_ack.hdr_dst_req.pval, - pckt->hdr.hdr_ack.hdr_send_offset, - pckt->hdr.hdr_common.hdr_flags & MCA_PML_BFO_HDR_FLAGS_NORDMA); - if( OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc) ) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.pckt_pending, - (opal_list_item_t*)pckt); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - return; - } - break; - case MCA_PML_BFO_HDR_TYPE_FIN: - rc = mca_pml_bfo_send_fin(pckt->proc, send_dst, - pckt->hdr.hdr_fin.hdr_des, - pckt->order, -#if PML_BFO - pckt->hdr.hdr_fin.hdr_fail, - pckt->hdr.hdr_fin.hdr_match.hdr_seq, - pckt->hdr.hdr_fin.hdr_match.hdr_common.hdr_flags, - pckt->hdr.hdr_fin.hdr_match.hdr_ctx, - pckt->hdr.hdr_fin.hdr_match.hdr_src); -#else /* PML_BFO */ - pckt->hdr.hdr_fin.hdr_fail); -#endif /* PML_BFO */ - if( OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc) ) { - return; - } - break; - default: - opal_output(0, "[%s:%d] wrong header type\n", - __FILE__, __LINE__); - break; - } - /* We're done with this packet, return it back to the free list */ - MCA_PML_BFO_PCKT_PENDING_RETURN(pckt); - } -} - -void mca_pml_bfo_process_pending_rdma(void) -{ - mca_pml_bfo_rdma_frag_t* frag; - int32_t i, rc, s = (int32_t)opal_list_get_size(&mca_pml_bfo.rdma_pending); - - for(i = 0; i < s; i++) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - frag = (mca_pml_bfo_rdma_frag_t*) - opal_list_remove_first(&mca_pml_bfo.rdma_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - if(NULL == frag) - break; - if(frag->rdma_state == MCA_PML_BFO_RDMA_PUT) { - frag->retries++; - rc = mca_pml_bfo_send_request_put_frag(frag); - } else { - rc = mca_pml_bfo_recv_request_get_frag(frag); - } - if(OMPI_ERR_OUT_OF_RESOURCE == rc) - break; - } -} - - -void mca_pml_bfo_error_handler( - struct mca_btl_base_module_t* btl, int32_t flags, - ompi_proc_t* errproc, char* btlinfo ) { -#if PML_BFO - if (flags & MCA_BTL_ERROR_FLAGS_NONFATAL) { - mca_pml_bfo_failover_error_handler(btl, flags, errproc, btlinfo); - return; - } -#endif /* PML_BFO */ - ompi_rte_abort(-1, NULL); -} - -#if OPAL_ENABLE_FT_CR == 0 -int mca_pml_bfo_ft_event( int state ) { - return OMPI_SUCCESS; -} -#else -int mca_pml_bfo_ft_event( int state ) -{ - static bool first_continue_pass = false; - ompi_proc_t** procs = NULL; - size_t num_procs; - int ret, p; - - if(OPAL_CRS_CHECKPOINT == state) { - if( opal_cr_timing_barrier_enabled ) { - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCPBR1); - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - } - - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P0); - } - else if(OPAL_CRS_CONTINUE == state) { - first_continue_pass = !first_continue_pass; - - if( !first_continue_pass ) { - if( opal_cr_timing_barrier_enabled ) { - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_COREBR0); - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - } - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P2); - } - - if (opal_cr_continue_like_restart && !first_continue_pass) { - /* - * Get a list of processes - */ - procs = ompi_proc_all(&num_procs); - if(NULL == procs) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* - * Refresh the proc structure, and publish our proc info in the modex. - * NOTE: Do *not* call ompi_proc_finalize as there are many places in - * the code that point to indv. procs in this strucutre. For our - * needs here we only need to fix up the modex, bml and pml - * references. - */ - if (OMPI_SUCCESS != (ret = ompi_proc_refresh())) { - opal_output(0, - "pml:bfo: ft_event(Restart): proc_refresh Failed %d", - ret); - for(p = 0; p < (int)num_procs; ++p) { - OBJ_RELEASE(procs[p]); - } - free (procs); - return ret; - } - } - } - else if(OPAL_CRS_RESTART_PRE == state ) { - /* Nothing here */ - } - else if(OPAL_CRS_RESTART == state ) { - /* - * Get a list of processes - */ - procs = ompi_proc_all(&num_procs); - if(NULL == procs) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* - * Clean out the modex information since it is invalid now. - * ompi_rte_purge_proc_attrs(); - * This happens at the ORTE level, so doing it again here will cause - * some issues with socket caching. - */ - - - /* - * Refresh the proc structure, and publish our proc info in the modex. - * NOTE: Do *not* call ompi_proc_finalize as there are many places in - * the code that point to indv. procs in this strucutre. For our - * needs here we only need to fix up the modex, bml and pml - * references. - */ - if (OMPI_SUCCESS != (ret = ompi_proc_refresh())) { - opal_output(0, - "pml:bfo: ft_event(Restart): proc_refresh Failed %d", - ret); - for(p = 0; p < (int)num_procs; ++p) { - OBJ_RELEASE(procs[p]); - } - free (procs); - return ret; - } - } - else if(OPAL_CRS_TERM == state ) { - ; - } - else { - ; - } - - /* Call the BML - * BML is expected to call ft_event in - * - BTL(s) - * - MPool(s) - */ - if( OMPI_SUCCESS != (ret = mca_bml.bml_ft_event(state))) { - opal_output(0, "pml:base: ft_event: BML ft_event function failed: %d\n", - ret); - } - - if(OPAL_CRS_CHECKPOINT == state) { - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P1); - - if( opal_cr_timing_barrier_enabled ) { - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR0); - /* JJH Cannot barrier here due to progress engine -- ompi_rte_barrier();*/ - } - } - else if(OPAL_CRS_CONTINUE == state) { - if( !first_continue_pass ) { - if( opal_cr_timing_barrier_enabled ) { - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR1); - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - } - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P3); - } - - if (opal_cr_continue_like_restart && !first_continue_pass) { - /* - * Exchange the modex information once again. - * BTLs will have republished their modex information. - */ - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - - /* - * Startup the PML stack now that the modex is running again - * Add the new procs (BTLs redo modex recv's) - */ - if( OMPI_SUCCESS != (ret = mca_pml_bfo_add_procs(procs, num_procs) ) ) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed in add_procs (%d)", ret); - return ret; - } - - /* Is this barrier necessary ? JJH */ - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - - if( NULL != procs ) { - for(p = 0; p < (int)num_procs; ++p) { - OBJ_RELEASE(procs[p]); - } - free(procs); - procs = NULL; - } - } - if( !first_continue_pass ) { - if( opal_cr_timing_barrier_enabled ) { - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR2); - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - } - OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCP1); - } - } - else if(OPAL_CRS_RESTART_PRE == state ) { - /* Nothing here */ - } - else if(OPAL_CRS_RESTART == state ) { - /* - * Exchange the modex information once again. - * BTLs will have republished their modex information. - */ - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - - /* - * Startup the PML stack now that the modex is running again - * Add the new procs (BTLs redo modex recv's) - */ - if( OMPI_SUCCESS != (ret = mca_pml_bfo_add_procs(procs, num_procs) ) ) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed in add_procs (%d)", ret); - return ret; - } - - /* Is this barrier necessary ? JJH */ - if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { - opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); - return ret; - } - - if( NULL != procs ) { - for(p = 0; p < (int)num_procs; ++p) { - OBJ_RELEASE(procs[p]); - } - free(procs); - procs = NULL; - } - } - else if(OPAL_CRS_TERM == state ) { - ; - } - else { - ; - } - - return OMPI_SUCCESS; -} -#endif /* OPAL_ENABLE_FT_CR */ - -int mca_pml_bfo_com_btl_comp(const void *v1, const void *v2) -{ - const mca_pml_bfo_com_btl_t *b1 = (const mca_pml_bfo_com_btl_t *) v1; - const mca_pml_bfo_com_btl_t *b2 = (const mca_pml_bfo_com_btl_t *) v2; - - if(b1->bml_btl->btl_weight < b2->bml_btl->btl_weight) - return 1; - if(b1->bml_btl->btl_weight > b2->bml_btl->btl_weight) - return -1; - - return 0; -} - diff --git a/ompi/mca/pml/bfo/pml_bfo.h b/ompi/mca/pml/bfo/pml_bfo.h deleted file mode 100644 index ef606f2669..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo.h +++ /dev/null @@ -1,362 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ - -#ifndef MCA_PML_BFO_H -#define MCA_PML_BFO_H - -#include "ompi_config.h" -#include "opal/class/opal_free_list.h" -#include "ompi/request/request.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/pml/base/pml_base_request.h" -#include "ompi/mca/pml/base/pml_base_bsend.h" -#include "ompi/mca/pml/base/pml_base_sendreq.h" -#include "ompi/datatype/ompi_datatype.h" -#include "pml_bfo_hdr.h" -#include "ompi/mca/bml/base/base.h" -#include "ompi/proc/proc.h" -#include "opal/mca/allocator/base/base.h" - -BEGIN_C_DECLS - -/** - * BFO PML module - */ - -struct mca_pml_bfo_t { - mca_pml_base_module_t super; - - int priority; - int free_list_num; /* initial size of free list */ - int free_list_max; /* maximum size of free list */ - int free_list_inc; /* number of elements to grow free list */ - unsigned int send_pipeline_depth; - unsigned int recv_pipeline_depth; - unsigned int rdma_put_retries_limit; - int max_rdma_per_request; - int max_send_per_range; - bool leave_pinned; - int leave_pinned_pipeline; - - /* lock queue access */ - opal_mutex_t lock; - - /* free lists */ - opal_free_list_t rdma_frags; - opal_free_list_t recv_frags; - opal_free_list_t pending_pckts; - opal_free_list_t buffers; - opal_free_list_t send_ranges; - - /* list of pending operations */ - opal_list_t pckt_pending; - opal_list_t send_pending; - opal_list_t recv_pending; - opal_list_t rdma_pending; - /* List of pending fragments without a matching communicator */ - opal_list_t non_existing_communicator_pending; - bool enabled; - char* allocator_name; - mca_allocator_base_module_t* allocator; - unsigned int unexpected_limit; -}; -typedef struct mca_pml_bfo_t mca_pml_bfo_t; - -extern mca_pml_bfo_t mca_pml_bfo; -extern int mca_pml_bfo_output; - -/* - * PML interface functions. - */ - -extern int mca_pml_bfo_add_comm( - struct ompi_communicator_t* comm -); - -extern int mca_pml_bfo_del_comm( - struct ompi_communicator_t* comm -); - -extern int mca_pml_bfo_add_procs( - struct ompi_proc_t **procs, - size_t nprocs -); - -extern int mca_pml_bfo_del_procs( - struct ompi_proc_t **procs, - size_t nprocs -); - -extern int mca_pml_bfo_enable( bool enable ); - -extern int mca_pml_bfo_progress(void); - -extern int mca_pml_bfo_iprobe( int dst, - int tag, - struct ompi_communicator_t* comm, - int *matched, - ompi_status_public_t* status ); - -extern int mca_pml_bfo_probe( int dst, - int tag, - struct ompi_communicator_t* comm, - ompi_status_public_t* status ); - -extern int mca_pml_bfo_improbe( int dst, - int tag, - struct ompi_communicator_t* comm, - int *matched, - struct ompi_message_t **message, - ompi_status_public_t* status ); - -extern int mca_pml_bfo_mprobe( int dst, - int tag, - struct ompi_communicator_t* comm, - struct ompi_message_t **message, - ompi_status_public_t* status ); - -extern int mca_pml_bfo_isend_init( void *buf, - size_t count, - ompi_datatype_t *datatype, - int dst, - int tag, - mca_pml_base_send_mode_t mode, - struct ompi_communicator_t* comm, - struct ompi_request_t **request ); - -extern int mca_pml_bfo_isend( void *buf, - size_t count, - ompi_datatype_t *datatype, - int dst, - int tag, - mca_pml_base_send_mode_t mode, - struct ompi_communicator_t* comm, - struct ompi_request_t **request ); - -extern int mca_pml_bfo_send( void *buf, - size_t count, - ompi_datatype_t *datatype, - int dst, - int tag, - mca_pml_base_send_mode_t mode, - struct ompi_communicator_t* comm ); - -extern int mca_pml_bfo_irecv_init( void *buf, - size_t count, - ompi_datatype_t *datatype, - int src, - int tag, - struct ompi_communicator_t* comm, - struct ompi_request_t **request ); - -extern int mca_pml_bfo_irecv( void *buf, - size_t count, - ompi_datatype_t *datatype, - int src, - int tag, - struct ompi_communicator_t* comm, - struct ompi_request_t **request ); - -extern int mca_pml_bfo_recv( void *buf, - size_t count, - ompi_datatype_t *datatype, - int src, - int tag, - struct ompi_communicator_t* comm, - ompi_status_public_t* status ); - -extern int mca_pml_bfo_imrecv( void *buf, - size_t count, - ompi_datatype_t *datatype, - struct ompi_message_t **message, - struct ompi_request_t **request ); - -extern int mca_pml_bfo_mrecv( void *buf, - size_t count, - ompi_datatype_t *datatype, - struct ompi_message_t **message, - ompi_status_public_t* status ); - -extern int mca_pml_bfo_dump( struct ompi_communicator_t* comm, - int verbose ); - -extern int mca_pml_bfo_start( size_t count, - ompi_request_t** requests ); - -extern int mca_pml_bfo_ft_event( int state ); - -END_C_DECLS - -struct mca_pml_bfo_pckt_pending_t { - opal_free_list_item_t super; - ompi_proc_t* proc; - mca_pml_bfo_hdr_t hdr; - struct mca_bml_base_btl_t *bml_btl; - uint8_t order; -}; -typedef struct mca_pml_bfo_pckt_pending_t mca_pml_bfo_pckt_pending_t; -OBJ_CLASS_DECLARATION(mca_pml_bfo_pckt_pending_t); - -#define MCA_PML_BFO_PCKT_PENDING_ALLOC(pckt) \ -do { \ - opal_free_list_item_t* item; \ - OPAL_FREE_LIST_WAIT(&mca_pml_bfo.pending_pckts, item); \ - pckt = (mca_pml_bfo_pckt_pending_t*)item; \ -} while (0) - -#define MCA_PML_BFO_PCKT_PENDING_RETURN(pckt) \ -do { \ - /* return packet */ \ - OPAL_FREE_LIST_RETURN(&mca_pml_bfo.pending_pckts, \ - (opal_free_list_item_t*)pckt); \ -} while(0) - -#define MCA_PML_BFO_ADD_FIN_TO_PENDING(P, D, B, O, S) \ - do { \ - mca_pml_bfo_pckt_pending_t *_pckt; \ - \ - MCA_PML_BFO_PCKT_PENDING_ALLOC(_pckt); \ - _pckt->hdr.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_FIN; \ - _pckt->hdr.hdr_fin.hdr_des = (D); \ - _pckt->hdr.hdr_fin.hdr_fail = (S); \ - _pckt->proc = (P); \ - _pckt->bml_btl = (B); \ - _pckt->order = (O); \ - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); \ - opal_list_append(&mca_pml_bfo.pckt_pending, \ - (opal_list_item_t*)_pckt); \ - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); \ - } while(0) - - -int mca_pml_bfo_send_fin(ompi_proc_t* proc, mca_bml_base_btl_t* bml_btl, -#if PML_BFO - opal_ptr_t hdr_des, uint8_t order, uint32_t status, - uint16_t seq, uint8_t reqseq, uint16_t ctx, uint32_t src); -#else /* PML_BFO */ - opal_ptr_t hdr_des, uint8_t order, uint32_t status); -#endif /* PML_BFO */ - -/* This function tries to resend FIN/ACK packets from pckt_pending queue. - * Packets are added to the queue when sending of FIN or ACK is failed due to - * resource unavailability. bml_btl passed to the function doesn't represents - * packet's destination, it represents BTL on which resource was freed, so only - * this BTL should be considered for resending packets */ -void mca_pml_bfo_process_pending_packets(mca_bml_base_btl_t* bml_btl); - -/* This function retries failed PUT/GET operations on frag. When RDMA operation - * cannot be accomplished for some reason, frag is put on the rdma_pending list. - * Later the operation is retried. The destination of RDMA operation is stored - * inside the frag structure */ -void mca_pml_bfo_process_pending_rdma(void); - -#define MCA_PML_BFO_PROGRESS_PENDING(bml_btl) \ - do { \ - if(opal_list_get_size(&mca_pml_bfo.pckt_pending)) \ - mca_pml_bfo_process_pending_packets(bml_btl); \ - if(opal_list_get_size(&mca_pml_bfo.recv_pending)) \ - mca_pml_bfo_recv_request_process_pending(); \ - if(opal_list_get_size(&mca_pml_bfo.send_pending)) \ - mca_pml_bfo_send_request_process_pending(bml_btl); \ - if(opal_list_get_size(&mca_pml_bfo.rdma_pending)) \ - mca_pml_bfo_process_pending_rdma(); \ - } while (0) - -/* - * Compute the total number of bytes on supplied descriptor - */ -static inline int mca_pml_bfo_compute_segment_length (size_t seg_size, void *segments, size_t count, - size_t hdrlen) { - size_t i, length; - - for (i = 0, length = -hdrlen ; i < count ; ++i) { - mca_btl_base_segment_t *segment = - (mca_btl_base_segment_t *)((char *) segments + i * seg_size); - - length += segment->seg_len; - } - - return length; -} - -static inline int mca_pml_bfo_compute_segment_length_base (mca_btl_base_segment_t *segments, - size_t count, size_t hdrlen) { - size_t i, length; - - for (i = 0, length = -hdrlen ; i < count ; ++i) { - length += segments[i].seg_len; - } - - return length; -} - -/* represent BTL chosen for sending request */ -struct mca_pml_bfo_com_btl_t { - mca_bml_base_btl_t *bml_btl; - struct mca_mpool_base_registration_t* btl_reg; - size_t length; -}; -typedef struct mca_pml_bfo_com_btl_t mca_pml_bfo_com_btl_t; - -int mca_pml_bfo_com_btl_comp(const void *v1, const void *v2); - -/* Calculate what percentage of a message to send through each BTL according to - * relative weight */ -static inline void -mca_pml_bfo_calc_weighted_length( mca_pml_bfo_com_btl_t *btls, int num_btls, size_t size, - double weight_total ) -{ - int i; - size_t length_left; - - /* shortcut for common case for only one BTL */ - if( OPAL_LIKELY(1 == num_btls) ) { - btls[0].length = size; - return; - } - - /* sort BTLs according of their weights so BTLs with smaller weight will - * not hijack all of the traffic */ - qsort( btls, num_btls, sizeof(mca_pml_bfo_com_btl_t), - mca_pml_bfo_com_btl_comp ); - - for(length_left = size, i = 0; i < num_btls; i++) { - mca_bml_base_btl_t* bml_btl = btls[i].bml_btl; - size_t length = 0; - if( OPAL_UNLIKELY(0 != length_left) ) { - length = (length_left > bml_btl->btl->btl_eager_limit)? - ((size_t)(size * (bml_btl->btl_weight / weight_total))) : - length_left; - - if(length > length_left) - length = length_left; - length_left -= length; - } - btls[i].length = length; - } - - /* account for rounding errors */ - btls[0].length += length_left; -} - -#endif diff --git a/ompi/mca/pml/bfo/pml_bfo_comm.c b/ompi/mca/pml/bfo/pml_bfo_comm.c deleted file mode 100644 index 997f191149..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_comm.c +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include - -#include "pml_bfo.h" -#include "pml_bfo_comm.h" - - - -static void mca_pml_bfo_comm_proc_construct(mca_pml_bfo_comm_proc_t* proc) -{ - proc->expected_sequence = 1; - proc->ompi_proc = NULL; - proc->send_sequence = 0; - OBJ_CONSTRUCT(&proc->frags_cant_match, opal_list_t); - OBJ_CONSTRUCT(&proc->specific_receives, opal_list_t); - OBJ_CONSTRUCT(&proc->unexpected_frags, opal_list_t); -} - - -static void mca_pml_bfo_comm_proc_destruct(mca_pml_bfo_comm_proc_t* proc) -{ - OBJ_DESTRUCT(&proc->frags_cant_match); - OBJ_DESTRUCT(&proc->specific_receives); - OBJ_DESTRUCT(&proc->unexpected_frags); -} - - -static OBJ_CLASS_INSTANCE( - mca_pml_bfo_comm_proc_t, - opal_object_t, - mca_pml_bfo_comm_proc_construct, - mca_pml_bfo_comm_proc_destruct); - - -static void mca_pml_bfo_comm_construct(mca_pml_bfo_comm_t* comm) -{ - OBJ_CONSTRUCT(&comm->wild_receives, opal_list_t); - OBJ_CONSTRUCT(&comm->matching_lock, opal_mutex_t); - comm->recv_sequence = 0; - comm->procs = NULL; - comm->last_probed = 0; - comm->num_procs = 0; -} - - -static void mca_pml_bfo_comm_destruct(mca_pml_bfo_comm_t* comm) -{ - size_t i; - for(i=0; inum_procs; i++) - OBJ_DESTRUCT((&comm->procs[i])); - if(NULL != comm->procs) - free(comm->procs); - OBJ_DESTRUCT(&comm->wild_receives); - OBJ_DESTRUCT(&comm->matching_lock); -} - - -OBJ_CLASS_INSTANCE( - mca_pml_bfo_comm_t, - opal_object_t, - mca_pml_bfo_comm_construct, - mca_pml_bfo_comm_destruct); - - -int mca_pml_bfo_comm_init_size(mca_pml_bfo_comm_t* comm, size_t size) -{ - size_t i; - - /* send message sequence-number support - sender side */ - comm->procs = (mca_pml_bfo_comm_proc_t*)malloc(sizeof(mca_pml_bfo_comm_proc_t)*size); - if(NULL == comm->procs) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - for(i=0; iprocs+i, mca_pml_bfo_comm_proc_t); - } - comm->num_procs = size; - return OMPI_SUCCESS; -} - - diff --git a/ompi/mca/pml/bfo/pml_bfo_comm.h b/ompi/mca/pml/bfo/pml_bfo_comm.h deleted file mode 100644 index c70b4514d3..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_comm.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ -#ifndef MCA_PML_BFO_COMM_H -#define MCA_PML_BFO_COMM_H - -#include "opal/threads/mutex.h" -#include "opal/class/opal_list.h" -#include "ompi/proc/proc.h" -BEGIN_C_DECLS - - -struct mca_pml_bfo_comm_proc_t { - opal_object_t super; - uint16_t expected_sequence; /**< send message sequence number - receiver side */ - struct ompi_proc_t* ompi_proc; -#if OPAL_ENABLE_MULTI_THREADS - volatile int32_t send_sequence; /**< send side sequence number */ -#else - int32_t send_sequence; /**< send side sequence number */ -#endif - opal_list_t frags_cant_match; /**< out-of-order fragment queues */ - opal_list_t specific_receives; /**< queues of unmatched specific receives */ - opal_list_t unexpected_frags; /**< unexpected fragment queues */ -}; -typedef struct mca_pml_bfo_comm_proc_t mca_pml_bfo_comm_proc_t; - - -/** - * Cached on ompi_communicator_t to hold queues/state - * used by the PML<->PTL interface for matching logic. - */ -struct mca_pml_comm_t { - opal_object_t super; -#if OPAL_ENABLE_MULTI_THREADS - volatile uint32_t recv_sequence; /**< recv request sequence number - receiver side */ -#else - uint32_t recv_sequence; /**< recv request sequence number - receiver side */ -#endif - opal_mutex_t matching_lock; /**< matching lock */ - opal_list_t wild_receives; /**< queue of unmatched wild (source process not specified) receives */ - mca_pml_bfo_comm_proc_t* procs; - size_t num_procs; - size_t last_probed; -}; -typedef struct mca_pml_comm_t mca_pml_bfo_comm_t; - -OBJ_CLASS_DECLARATION(mca_pml_bfo_comm_t); - - -/** - * Initialize an instance of mca_pml_bfo_comm_t based on the communicator size. - * - * @param comm Instance of mca_pml_bfo_comm_t - * @param size Size of communicator - * @return OMPI_SUCCESS or error status on failure. - */ - -extern int mca_pml_bfo_comm_init_size(mca_pml_bfo_comm_t* comm, size_t size); - -END_C_DECLS -#endif - diff --git a/ompi/mca/pml/bfo/pml_bfo_component.c b/ompi/mca/pml/bfo/pml_bfo_component.c deleted file mode 100644 index 67e5927261..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_component.c +++ /dev/null @@ -1,274 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2014 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "opal/mca/event/event.h" -#include "mpi.h" -#include "ompi/runtime/params.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/pml/base/pml_base_bsend.h" -#include "pml_bfo.h" -#include "pml_bfo_hdr.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_rdmafrag.h" -#include "pml_bfo_recvfrag.h" -#include "ompi/mca/bml/base/base.h" -#include "pml_bfo_component.h" -#include "opal/mca/allocator/base/base.h" -#include "opal/runtime/opal_params.h" - -OBJ_CLASS_INSTANCE( mca_pml_bfo_pckt_pending_t, - ompi_free_list_item_t, - NULL, - NULL ); - -static int mca_pml_bfo_component_register(void); -static int mca_pml_bfo_component_open(void); -static int mca_pml_bfo_component_close(void); -static mca_pml_base_module_t* -mca_pml_bfo_component_init( int* priority, bool enable_progress_threads, - bool enable_mpi_threads ); -static int mca_pml_bfo_component_fini(void); -int mca_pml_bfo_output = 0; -static int mca_pml_bfo_verbose = 0; - -mca_pml_base_component_2_0_0_t mca_pml_bfo_component = { - - /* First, the mca_base_component_t struct containing meta - information about the component itself */ - - .pmlm_version = { - MCA_PML_BASE_VERSION_2_0_0, - - .mca_component_name = "bfo", - MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION, - OMPI_RELEASE_VERSION), - .mca_open_component = mca_pml_bfo_component_open, - .mca_close_component = mca_pml_bfo_component_close, - .mca_register_component_params = mca_pml_bfo_component_register, - }, - .pmlm_data = { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - }, - - .pmlm_init = mca_pml_bfo_component_init, - .pmlm_finalize = mca_pml_bfo_component_fini, -}; - -void *mca_pml_bfo_seg_alloc( struct mca_mpool_base_module_t* mpool, - size_t* size, - mca_mpool_base_registration_t** registration); - -void mca_pml_bfo_seg_free( struct mca_mpool_base_module_t* mpool, - void* segment ); - -static inline int mca_pml_bfo_param_register_int( - const char* param_name, - int default_value, - int *storage) -{ - *storage = default_value; - (void) mca_base_component_var_register(&mca_pml_bfo_component.pmlm_version, param_name, - NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, storage); - - return *storage; -} - -static inline unsigned int mca_pml_bfo_param_register_uint( - const char* param_name, - unsigned int default_value, - unsigned int *storage) -{ - *storage = default_value; - (void) mca_base_component_var_register(&mca_pml_bfo_component.pmlm_version, param_name, - NULL, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, storage); - - return *storage; -} - -static int mca_pml_bfo_component_register(void) -{ - int default_priority; - -#if PML_BFO - default_priority = 5; -#else /* PML_BFO */ - default_priority = 20; - mca_pml_bfo_param_register_int("priority", 20); -#endif /* PML_BFO */ - - (void) mca_pml_bfo_param_register_int("verbose", 0, &mca_pml_bfo_verbose); - (void) mca_pml_bfo_param_register_int("free_list_num", 4, &mca_pml_bfo.free_list_num); - (void) mca_pml_bfo_param_register_int("free_list_max", -1, &mca_pml_bfo.free_list_max); - (void) mca_pml_bfo_param_register_int("free_list_inc", 64, &mca_pml_bfo.free_list_inc); - (void) mca_pml_bfo_param_register_int("priority", default_priority, &mca_pml_bfo.priority); - (void) mca_pml_bfo_param_register_uint("send_pipeline_depth", 3, &mca_pml_bfo.send_pipeline_depth); - (void) mca_pml_bfo_param_register_uint("recv_pipeline_depth", 4, &mca_pml_bfo.recv_pipeline_depth); - (void) mca_pml_bfo_param_register_uint("rdma_put_retries_limit", 5, &mca_pml_bfo.rdma_put_retries_limit); - (void) mca_pml_bfo_param_register_int("max_rdma_per_request", 4, &mca_pml_bfo.max_rdma_per_request); - (void) mca_pml_bfo_param_register_int("max_send_per_range", 4, &mca_pml_bfo.max_send_per_range); - (void) mca_pml_bfo_param_register_uint("unexpected_limit", 128, &mca_pml_bfo.unexpected_limit); - - mca_pml_bfo.allocator_name = "bucket"; - (void) mca_base_component_var_register(&mca_pml_bfo_component.pmlm_version, - "allocator", - "Name of allocator component for unexpected messages", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_pml_bfo.allocator_name); - - return OMPI_SUCCESS; -} - -static int mca_pml_bfo_component_open(void) -{ - mca_pml_bfo_output = opal_output_open(NULL); - opal_output_set_verbosity(mca_pml_bfo_output, mca_pml_bfo_verbose); - - mca_pml_bfo.enabled = false; - return mca_base_framework_open(&ompi_bml_base_framework, 0); -} - - -static int mca_pml_bfo_component_close(void) -{ - int rc; - - if (OMPI_SUCCESS != (rc = mca_base_framework_close(&ompi_bml_base_framework))) { - return rc; - } - opal_output_close(mca_pml_bfo_output); - - return OMPI_SUCCESS; -} - - -static mca_pml_base_module_t* -mca_pml_bfo_component_init( int* priority, - bool enable_progress_threads, - bool enable_mpi_threads ) -{ - mca_allocator_base_component_t* allocator_component; - - opal_output_verbose( 10, mca_pml_bfo_output, - "in bfo, my priority is %d\n", mca_pml_bfo.priority); - - if((*priority) > mca_pml_bfo.priority) { - *priority = mca_pml_bfo.priority; - return NULL; - } - *priority = mca_pml_bfo.priority; - - allocator_component = mca_allocator_component_lookup( mca_pml_bfo.allocator_name ); - if(NULL == allocator_component) { - opal_output(0, "mca_pml_bfo_component_init: can't find allocator: %s\n", mca_pml_bfo.allocator_name); - return NULL; - } - - mca_pml_bfo.allocator = allocator_component->allocator_init(true, - mca_pml_bfo_seg_alloc, - mca_pml_bfo_seg_free, NULL); - if(NULL == mca_pml_bfo.allocator) { - opal_output(0, "mca_pml_bfo_component_init: unable to initialize allocator\n"); - return NULL; - } - - - if(OMPI_SUCCESS != mca_bml_base_init( enable_progress_threads, - enable_mpi_threads)) { - return NULL; - } - - /* Set this here (vs in component_open()) because - opal_leave_pinned* may have been set after MCA params were - read (e.g., by the openib btl) */ - mca_pml_bfo.leave_pinned = (1 == opal_leave_pinned); - mca_pml_bfo.leave_pinned_pipeline = (int) opal_leave_pinned_pipeline; - - return &mca_pml_bfo.super; -} - -int mca_pml_bfo_component_fini(void) -{ - int rc; - - /* Shutdown BML */ - if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize())) - return rc; - - if(!mca_pml_bfo.enabled) - return OMPI_SUCCESS; /* never selected.. return success.. */ - mca_pml_bfo.enabled = false; /* not anymore */ - - OBJ_DESTRUCT(&mca_pml_bfo.rdma_pending); - OBJ_DESTRUCT(&mca_pml_bfo.pckt_pending); - OBJ_DESTRUCT(&mca_pml_bfo.recv_pending); - OBJ_DESTRUCT(&mca_pml_bfo.send_pending); - OBJ_DESTRUCT(&mca_pml_bfo.non_existing_communicator_pending); - OBJ_DESTRUCT(&mca_pml_bfo.buffers); - OBJ_DESTRUCT(&mca_pml_bfo.pending_pckts); - OBJ_DESTRUCT(&mca_pml_bfo.recv_frags); - OBJ_DESTRUCT(&mca_pml_bfo.rdma_frags); - OBJ_DESTRUCT(&mca_pml_bfo.lock); - - if(OMPI_SUCCESS != (rc = mca_pml_bfo.allocator->alc_finalize(mca_pml_bfo.allocator))) { - return rc; - } - -#if 0 - if (mca_pml_base_send_requests.fl_num_allocated != - mca_pml_base_send_requests.super.opal_list_length) { - opal_output(0, "bfo send requests: %d allocated %d returned\n", - mca_pml_base_send_requests.fl_num_allocated, - mca_pml_base_send_requests.super.opal_list_length); - } - if (mca_pml_base_recv_requests.fl_num_allocated != - mca_pml_base_recv_requests.super.opal_list_length) { - opal_output(0, "bfo recv requests: %d allocated %d returned\n", - mca_pml_base_recv_requests.fl_num_allocated, - mca_pml_base_recv_requests.super.opal_list_length); - } -#endif - - return OMPI_SUCCESS; -} - -void *mca_pml_bfo_seg_alloc( struct mca_mpool_base_module_t* mpool, - size_t* size, - mca_mpool_base_registration_t** registration) { - return malloc(*size); -} - -void mca_pml_bfo_seg_free( struct mca_mpool_base_module_t* mpool, - void* segment ) { - free(segment); -} diff --git a/ompi/mca/pml/bfo/pml_bfo_component.h b/ompi/mca/pml/bfo/pml_bfo_component.h deleted file mode 100644 index 60a9828a54..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_component.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ - -#ifndef MCA_PML_BFO_COMPONENT_H -#define MCA_PML_BFO_COMPONENT_H - -BEGIN_C_DECLS - -/* - * PML module functions. - */ -OMPI_MODULE_DECLSPEC extern mca_pml_base_component_2_0_0_t mca_pml_bfo_component; - -END_C_DECLS - -#endif diff --git a/ompi/mca/pml/bfo/pml_bfo_cuda.c b/ompi/mca/pml/bfo/pml_bfo_cuda.c deleted file mode 100644 index eb35b226e0..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_cuda.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012-2015 NVIDIA Corporation. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "opal/prefetch.h" -#include "opal/mca/btl/btl.h" -#include "opal/mca/mpool/mpool.h" -#include "ompi/constants.h" -#include "ompi/mca/pml/pml.h" -#include "pml_bfo.h" -#include "pml_bfo_hdr.h" -#include "pml_bfo_rdmafrag.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_sendreq.h" -#include "ompi/mca/bml/base/base.h" -#include "ompi/memchecker.h" - -size_t mca_pml_bfo_rdma_cuda_btls( - mca_bml_base_endpoint_t* bml_endpoint, - unsigned char* base, - size_t size, - mca_pml_bfo_com_btl_t* rdma_btls); - -int mca_pml_bfo_cuda_need_buffers(void * rreq, - mca_btl_base_module_t* btl); - -/** - * Handle the CUDA buffer. - */ -int mca_pml_bfo_send_request_start_cuda(mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size) { - int rc; - sendreq->req_send.req_base.req_convertor.flags &= ~CONVERTOR_CUDA; - if (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) { - unsigned char *base; - opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base ); - /* Set flag back */ - sendreq->req_send.req_base.req_convertor.flags |= CONVERTOR_CUDA; - if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_bfo_rdma_cuda_btls( - sendreq->req_endpoint, - base, - sendreq->req_send.req_bytes_packed, - sendreq->req_rdma))) { - rc = mca_pml_bfo_send_request_start_rdma(sendreq, bml_btl, - sendreq->req_send.req_bytes_packed); - if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { - mca_pml_bfo_free_rdma_resources(sendreq); - } - } else { - if (bml_btl->btl_flags & MCA_BTL_FLAGS_CUDA_PUT) { - rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, - MCA_PML_BFO_HDR_FLAGS_CONTIG); - } else { - rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0); - } - } - } else { - /* Do not send anything with first rendezvous message as copying GPU - * memory into RNDV message is expensive. */ - sendreq->req_send.req_base.req_convertor.flags |= CONVERTOR_CUDA; - rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, 0, 0); - } - return rc; -} - - - -size_t mca_pml_bfo_rdma_cuda_btls( - mca_bml_base_endpoint_t* bml_endpoint, - unsigned char* base, - size_t size, - mca_pml_bfo_com_btl_t* rdma_btls) -{ - int num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_send); - double weight_total = 0; - int num_btls_used = 0, n; - - /* shortcut when there are no rdma capable btls */ - if(num_btls == 0) { - return 0; - } - - /* check to see if memory is registered */ - for(n = 0; n < num_btls && num_btls_used < mca_pml_bfo.max_rdma_per_request; - n++) { - mca_bml_base_btl_t* bml_btl = - mca_bml_base_btl_array_get_index(&bml_endpoint->btl_send, n); - - if (bml_btl->btl_flags & MCA_BTL_FLAGS_CUDA_GET) { - mca_mpool_base_registration_t* reg = NULL; - mca_mpool_base_module_t *btl_mpool = bml_btl->btl->btl_mpool; - - if( NULL != btl_mpool ) { - /* register the memory */ - btl_mpool->mpool_register(btl_mpool, base, size, 0, ®); - } - - if(NULL == reg) - continue; - - rdma_btls[num_btls_used].bml_btl = bml_btl; - rdma_btls[num_btls_used].btl_reg = reg; - weight_total += bml_btl->btl_weight; - num_btls_used++; - } - } - - /* if we don't use leave_pinned and all BTLs that already have this memory - * registered amount to less then half of available bandwidth - fall back to - * pipeline protocol */ - if(0 == num_btls_used || (!mca_pml_bfo.leave_pinned && weight_total < 0.5)) - return 0; - - mca_pml_bfo_calc_weighted_length(rdma_btls, num_btls_used, size, - weight_total); - - return num_btls_used; -} - -int mca_pml_bfo_cuda_need_buffers(void * rreq, - mca_btl_base_module_t* btl) -{ - mca_pml_bfo_recv_request_t* recvreq = (mca_pml_bfo_recv_request_t*)rreq; - if ((recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) && - (btl->btl_flags & MCA_BTL_FLAGS_CUDA_GET)) { - recvreq->req_recv.req_base.req_convertor.flags &= ~CONVERTOR_CUDA; - if(opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == true) { - recvreq->req_recv.req_base.req_convertor.flags |= CONVERTOR_CUDA; - return true; - } else { - recvreq->req_recv.req_base.req_convertor.flags |= CONVERTOR_CUDA; - return false; - } - } - return true; -} - diff --git a/ompi/mca/pml/bfo/pml_bfo_failover.c b/ompi/mca/pml/bfo/pml_bfo_failover.c deleted file mode 100644 index bd8e3a5452..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_failover.c +++ /dev/null @@ -1,2187 +0,0 @@ -/* - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011-2012 Los Alamos National Security, LLC. - * All rights reserved. - * Copyright (c) 2013 Intel, Inc. All rights reserved - * Copyright (c) 2014 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/** - * @file - * Functions that implement failover capabilities. To utilize the - * failover feature, one needs to configure the library with - * --enable-openib-failover. Then the system that is being used - * must have two or more openib BTLs in use. When an error occurs, - * the BTL will call into this PML to map out the offending BTL and - * continue using the one that is still working. - * Most of the differences between the ob1 PML and the bfo PML are - * contained in this file. - */ - -#include "ompi_config.h" - -#include -#include - -#include "opal/class/opal_bitmap.h" -#include "opal/mca/btl/btl.h" -#include "opal/mca/btl/base/base.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/pml/base/base.h" -#include "ompi/mca/pml/base/base.h" -#include "pml_bfo.h" -#include "pml_bfo_component.h" -#include "pml_bfo_comm.h" -#include "pml_bfo_hdr.h" -#include "pml_bfo_recvfrag.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_rdmafrag.h" -#include "pml_bfo_failover.h" -#include "ompi/mca/bml/base/base.h" - -#include "ompi/runtime/ompi_cr.h" - -static void mca_pml_bfo_error_pending_packets(mca_btl_base_module_t* btl, - mca_bml_base_endpoint_t* ep); - -/** - * When running with failover enabled, check the PML sequence numbers - * to see if we have received a duplicate message. This check is done - * for for all MATCH fragments. It is also done for RNDV and RGET - * fragments that do not have the MCA_PML_BFO_HDR_FLAGS_RESTART flag - * set. - * We set the window size to half the total range of sequence numbers. - * We only enter this code when the seq_num is not the expected one. - * A few more notes on the algorithm used here. In normal operation, - * the expected value will either be equal to or less than the - * sequence number of the header. This is because we are using this - * sequence number to detect packets arriving prior to them being - * expected. If we determine that expected is less than header, then - * make sure this is not a rollover case. We do that by adding the - * maxnum to the expected. - * @param proc Pointer to proc from where message came - * @param hdr Pointer to header of message - */ -bool mca_pml_bfo_is_duplicate_msg(mca_pml_bfo_comm_proc_t* proc, - mca_pml_bfo_match_hdr_t *hdr) -{ - const int window = 32768; - const int maxnum = 65536; - mca_pml_bfo_recv_frag_t *frag; - -#if 0 - opal_output(0, "checking dup, exp=%d, act=%d, type=%d, cant_match=%d\n", - (uint16_t)proc->expected_sequence, - hdr->hdr_seq, hdr->hdr_common.hdr_type, - opal_list_get_size(&proc->frags_cant_match)); -#endif - - /* Few cases near end of values where expected may equal 65535 and - * an out of order shows up that may equal something like 1. */ - if (OPAL_UNLIKELY((uint16_t)proc->expected_sequence > hdr->hdr_seq)) { - if (((uint16_t)proc->expected_sequence - hdr->hdr_seq) < window) { - opal_output_verbose(20, mca_pml_bfo_output, - "%s:%d: frag duplicated, exp=%d, act=%d, type=%d\n", - __FILE__, __LINE__, (uint16_t)proc->expected_sequence, - hdr->hdr_seq, hdr->hdr_common.hdr_type); - return true; - } - } else { - /* This is the normal flow through this code. We also need to - * use the maxnum to ensure that we handle cases where the - * expected number has rolled over but then a duplicate message - * shows up that is greater than it. */ - if ((((uint16_t)proc->expected_sequence + maxnum) - hdr->hdr_seq) < window) { - opal_output_verbose(20, mca_pml_bfo_output, - "%s:%d: frag duplicated, exp=%d, act=%d, type=%d\n", - __FILE__, __LINE__, (uint16_t)proc->expected_sequence, - hdr->hdr_seq, hdr->hdr_common.hdr_type); - return true; - } - } - - /* Need to explicitly check against any out of order fragments. Unfortunately, we - * always have to do this since we can get a duplicate out of order fragment. */ - if(OPAL_UNLIKELY(opal_list_get_size(&proc->frags_cant_match) > 0)) { - for(frag = (mca_pml_bfo_recv_frag_t*)opal_list_get_first(&proc->frags_cant_match); - frag != (mca_pml_bfo_recv_frag_t*)opal_list_get_end(&proc->frags_cant_match); - frag = (mca_pml_bfo_recv_frag_t*)opal_list_get_next(frag)) - { - mca_pml_bfo_match_hdr_t* mhdr = &frag->hdr.hdr_match; - - if(mhdr->hdr_seq == hdr->hdr_seq) { - opal_output_verbose(20, mca_pml_bfo_output, - "%s:%d: frag duplicated on frags_cant_match list, seq=%d, type=%d\n", - __FILE__, __LINE__, hdr->hdr_seq, hdr->hdr_common.hdr_type); - return true; - } - } - } - - return false; -} - -/** - * This function checks to see if we have received a duplicate FIN - * message. This is done by first pulling the pointer of the request - * that the FIN message is pointing to from the message. We then - * check the various fields in the request to the fields in the header - * and make sure they match. If they do not, then the request must - * have been recycled already and this is a duplicate FIN message. We - * have to do this check on every FIN message that we receive. - */ -bool mca_pml_bfo_is_duplicate_fin(mca_pml_bfo_hdr_t* hdr, mca_btl_base_descriptor_t* rdma, - mca_btl_base_module_t* btl) -{ - mca_pml_base_request_t* basereq; - /* When running with failover enabled, need to ensure that this - * is not a duplicate FIN message. */ - if (btl->btl_flags & MCA_BTL_FLAGS_FAILOVER_SUPPORT) { - /* The first check is to make sure the descriptor is pointing - * to a valid request. The descriptor may be pointing to NULL - * if it was freed and not reused yet. */ - if (NULL == rdma->des_cbdata) { - opal_output_verbose(20, mca_pml_bfo_output, - "FIN: received: dropping because not pointing to valid descriptor " - "PML=%d CTX=%d SRC=%d RQS=%d", - hdr->hdr_fin.hdr_match.hdr_seq, - hdr->hdr_fin.hdr_match.hdr_ctx, - hdr->hdr_fin.hdr_match.hdr_src, - hdr->hdr_fin.hdr_match.hdr_common.hdr_flags); - return true; - } - - basereq = (mca_pml_base_request_t*)rdma->des_cbdata; - /* Now we know the descriptor is pointing to a non-null request. - * Does it match what we expect? To make sure the receiver request - * matches the FIN message, check the context number, source of the - * message, and MPI sequence number. Then make sure that it also - * matches the internal sequencing number of the requests. We need - * to look at the type of request we are pointing at to figure out - * what fields to access. */ - if (basereq->req_type == MCA_PML_REQUEST_RECV) { - mca_pml_bfo_recv_request_t* recvreq = (mca_pml_bfo_recv_request_t*)basereq; - if ((hdr->hdr_fin.hdr_match.hdr_ctx != - recvreq->req_recv.req_base.req_comm->c_contextid) || - (hdr->hdr_fin.hdr_match.hdr_src != - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE) || - (hdr->hdr_fin.hdr_match.hdr_seq != (uint16_t)recvreq->req_msgseq)) { - opal_output_verbose(5, mca_pml_bfo_output, - "FIN: received on receiver: dropping because no match " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d " - "RQS:exp=%d,act=%d, dst_req=%p", - (uint16_t)recvreq->req_msgseq, hdr->hdr_fin.hdr_match.hdr_seq, - recvreq->req_recv.req_base.req_comm->c_contextid, - hdr->hdr_fin.hdr_match.hdr_ctx, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - hdr->hdr_fin.hdr_match.hdr_src, - recvreq->req_restartseq, - hdr->hdr_fin.hdr_match.hdr_common.hdr_flags, - (void *)recvreq); - return true; - } - if (hdr->hdr_fin.hdr_match.hdr_common.hdr_flags != recvreq->req_restartseq) { - opal_output_verbose(5, mca_pml_bfo_output, - "FIN: received on receiver: dropping because old " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d " - "RQS:exp=%d,act=%d, dst_req=%p", - (uint16_t)recvreq->req_msgseq, hdr->hdr_fin.hdr_match.hdr_seq, - recvreq->req_recv.req_base.req_comm->c_contextid, - hdr->hdr_fin.hdr_match.hdr_ctx, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - hdr->hdr_fin.hdr_match.hdr_src, - recvreq->req_restartseq, - hdr->hdr_fin.hdr_match.hdr_common.hdr_flags, - (void *)recvreq); - return true; - } - } else if (basereq->req_type == MCA_PML_REQUEST_SEND) { - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)basereq; - if ((hdr->hdr_fin.hdr_match.hdr_ctx != - sendreq->req_send.req_base.req_comm->c_contextid) || - (hdr->hdr_fin.hdr_match.hdr_src != - sendreq->req_send.req_base.req_peer) || - (hdr->hdr_fin.hdr_match.hdr_seq != - (uint16_t)sendreq->req_send.req_base.req_sequence)) { - uint16_t seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - opal_output_verbose(5, mca_pml_bfo_output, - "FIN: received on sender: dropping because no match " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d " - "RQS:exp=%d,act=%d, dst_req=%p", - seq, hdr->hdr_fin.hdr_match.hdr_seq, - sendreq->req_send.req_base.req_comm->c_contextid, - hdr->hdr_fin.hdr_match.hdr_ctx, - sendreq->req_send.req_base.req_peer, - hdr->hdr_fin.hdr_match.hdr_src, - sendreq->req_restartseq, - hdr->hdr_fin.hdr_match.hdr_common.hdr_flags, - (void *)sendreq); - return true; - } - if (hdr->hdr_fin.hdr_match.hdr_common.hdr_flags != sendreq->req_restartseq) { - uint16_t seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - opal_output_verbose(5, mca_pml_bfo_output, - "FIN: received on sender: dropping because old " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d " - "RQS:exp=%d,act=%d, dst_req=%p", - seq, hdr->hdr_fin.hdr_match.hdr_seq, - sendreq->req_send.req_base.req_comm->c_contextid, - hdr->hdr_fin.hdr_match.hdr_ctx, - sendreq->req_send.req_base.req_peer, - hdr->hdr_fin.hdr_match.hdr_src, - sendreq->req_restartseq, - hdr->hdr_fin.hdr_match.hdr_common.hdr_flags, - (void *)sendreq); - return true; - } - } else { - /* We can get here if the descriptor has been reused, but - * not as an RDMA descriptor. In that case, the callback - * function has been set to something else. Clearly the - * descriptor we are interested is gone, so just drop the - * FIN message. */ - opal_output_verbose(5, mca_pml_bfo_output, - "FIN: received: dropping because descriptor has been reused " - "PML=%d CTX=%d SRC=%d RQS=%d rdma->des_flags=%d", - hdr->hdr_fin.hdr_match.hdr_seq, hdr->hdr_fin.hdr_match.hdr_ctx, - hdr->hdr_fin.hdr_match.hdr_src, hdr->hdr_fin.hdr_match.hdr_common.hdr_flags, - rdma->des_flags); - return true; - } - } - return false; -} - -/** - * Repost a FIN message if we get an error on the completion event. - */ -void mca_pml_bfo_repost_fin(struct mca_btl_base_descriptor_t* des) { - /* In the error case, we will repost the FIN message. I had - * considered restarting the request. The problem is that the - * request may be already complete when we detect that a FIN - * message got an error on its completion event. For example, with - * the PUT protocol, if the RDMA writes succeed and all the data - * has been sent, then the request is marked as complete and can be - * freed. Therefore, an error on the FIN message has no request to - * refer back to. So, we will just repost it. However, we are also - * faced with the case where the FIN message has an error but it - * actually makes it to the other side. In that case we are now - * sending a FIN message to a non-existent request on the receiver - * side. To handle that, we have added the match information to - * the FIN message. That way, we can check on the receiving side - * to ensure that it is pointing to a valid request. */ - mca_pml_bfo_fin_hdr_t* hdr; - mca_bml_base_endpoint_t* bml_endpoint; - ompi_proc_t *proc; - mca_bml_base_btl_t* bml_btl; - - proc = (ompi_proc_t*) des->des_cbdata; - bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - hdr = (mca_pml_bfo_fin_hdr_t*)des->des_local->seg_addr.pval; - - opal_output_verbose(20, mca_pml_bfo_output, - "REPOST: BFO_HDR_TYPE_FIN: seq=%d,myrank=%d,peer=%d,hdr->hdr_fail=%d,src=%d", - hdr->hdr_match.hdr_seq, OMPI_PROC_MY_NAME->vpid, OMPI_CAST_RTE_NAME(&proc->super.proc_name)->vpid, - hdr->hdr_fail, hdr->hdr_match.hdr_src); - - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - - /* Reconstruct the fin for sending on the other BTL */ - mca_pml_bfo_send_fin(proc, bml_btl, - hdr->hdr_des, MCA_BTL_NO_ORDER, - hdr->hdr_fail, hdr->hdr_match.hdr_seq, - hdr->hdr_match.hdr_common.hdr_flags, - hdr->hdr_match.hdr_ctx, hdr->hdr_match.hdr_src); - return; -} - -/** - * This function is called when a RNDV or RGET is received with the - * FLAGS_RESTART flag set. This means this message already has a - * receive request already associated with it. - */ -mca_pml_bfo_recv_request_t* mca_pml_bfo_get_request(mca_pml_bfo_match_hdr_t *hdr) { - mca_pml_bfo_recv_request_t *match = NULL; - mca_pml_bfo_rendezvous_hdr_t * rhdr = (mca_pml_bfo_rendezvous_hdr_t *) hdr; - match = (mca_pml_bfo_recv_request_t *) rhdr->hdr_dst_req.pval; - - /* Check to see if we have received a duplicate RNDV (or RGET). This can - * occur because we got an error when we reposted the RNDV. Therefore, - * we make sure that the request has not completed from underneath us - * and been recycled. Secondly, make sure we are not getting it a - * second time for the same request. */ - if ((rhdr->hdr_match.hdr_ctx != match->req_recv.req_base.req_comm->c_contextid) || - (rhdr->hdr_match.hdr_src != match->req_recv.req_base.req_ompi.req_status.MPI_SOURCE) || - (rhdr->hdr_match.hdr_seq != (uint16_t)match->req_msgseq) || - (rhdr->hdr_restartseq == match->req_restartseq)) { - if (hdr->hdr_common.hdr_type == MCA_PML_BFO_HDR_TYPE_RNDV) { - opal_output_verbose(20, mca_pml_bfo_output, - "RNDV: received with RESTART flag: duplicate, dropping " - "PML:exp=%d,act=%d RQS=%d, src_req=%p, dst_req=%p, peer=%d", - match->req_msgseq, rhdr->hdr_match.hdr_seq, match->req_restartseq, - match->remote_req_send.pval, (void *)match, - match->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - } else { - opal_output_verbose(20, mca_pml_bfo_output, - "RGET: received with RESTART flag: duplicate, dropping " - "PML:exp=%d,act=%d RQS=%d, src_req=%p, dst_req=%p, peer=%d", - match->req_msgseq, rhdr->hdr_match.hdr_seq, match->req_restartseq, - match->remote_req_send.pval, (void *)match, - match->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - } - return NULL; - } - - mca_pml_bfo_recv_request_reset(match); - if (hdr->hdr_common.hdr_type == MCA_PML_BFO_HDR_TYPE_RNDV) { - opal_output_verbose(30, mca_pml_bfo_output, - "RNDV: received with RESTART flag: restarting recv, " - "PML:exp=%d,act=%d RQS(new)=%d, src_req=%p, dst_req=%p, peer=%d", - match->req_msgseq, rhdr->hdr_match.hdr_seq, match->req_restartseq, - match->remote_req_send.pval, (void *)match, - match->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - } else { - opal_output_verbose(30, mca_pml_bfo_output, - "RGET: received with RESTART flag: restarting recv, " - "PML:exp=%d,act=%d RQS(new)=%d, src_req=%p, dst_req=%p, peer=%d", - match->req_msgseq, rhdr->hdr_match.hdr_seq, match->req_restartseq, - match->remote_req_send.pval, (void *)match, - match->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - } - return match; -} - -/** - * Callback for when a RNDVRESTARTNOTIFY message is received. A - * RNDVRESTARTNOTIFY message is sent from the sender to the receiver - * telling the receiver that the message is going to be started over. - * The receiver first makes sure that the request being pointed to is - * still valid. If it is not, that means the receiver must have - * completed the request and therefore we need to send a NACK back to - * the sender. The receiver then makes sure this is not a duplicate - * message. If it is a duplicate, it will just drop it. Otherwise, - * it will then send a RNDVRESTARTACK message if there are no - * outstanding events on the receiver. Otherwise, it will just change - * the state of the request and wait for another event to send the - * RNDVRESTARTACK to the sender. - */ -void mca_pml_bfo_recv_frag_callback_rndvrestartnotify(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_recv_request_t* recvreq; - ompi_proc_t* ompi_proc; - ompi_process_name_t orte_proc; - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY); - recvreq = (mca_pml_bfo_recv_request_t*)hdr->hdr_restart.hdr_dst_req.pval; - - /* Check to see if the receive request is still valid. If the - * request is recycled, that means the original request must have - * completed and we therefore need to send a NACK back to the sender. - * Note that when the request is gone, we need to pull some information - * off the header so that we can figure out where to send the NACK - * message back to. */ - if ((hdr->hdr_match.hdr_ctx != recvreq->req_recv.req_base.req_comm->c_contextid) || - (hdr->hdr_match.hdr_src != recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE) || - (hdr->hdr_match.hdr_seq != (uint16_t)recvreq->req_msgseq)) { - orte_proc.jobid = hdr->hdr_restart.hdr_jobid; - orte_proc.vpid = hdr->hdr_restart.hdr_vpid; - - ompi_proc = ompi_proc_find(&orte_proc); - opal_output_verbose(20, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: received: does not match request, sending NACK back " - "PML:req=%d,hdr=%d CTX:req=%d,hdr=%d SRC:req=%d,hdr=%d " - "RQS:req=%d,hdr=%d src_req=%p, dst_req=%p, peer=%d, hdr->hdr_jobid=%d, " - "hdr->hdr_vpid=%d, proc_hostname=%s", - (uint16_t)recvreq->req_msgseq, hdr->hdr_match.hdr_seq, - recvreq->req_recv.req_base.req_comm->c_contextid, hdr->hdr_match.hdr_ctx, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - hdr->hdr_match.hdr_src, recvreq->req_restartseq, - hdr->hdr_restart.hdr_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - hdr->hdr_restart.hdr_jobid, hdr->hdr_restart.hdr_vpid, - (NULL == ompi_proc->super.proc_hostname) ? "unknown" : ompi_proc->super.proc_hostname); - mca_pml_bfo_recv_request_rndvrestartnack(des, ompi_proc, false); - return; - } - - /* We know that we have the correct receive request. Make sure this is not - * a duplicate RNDVRESTARTNOTIFY on this request. */ - if (hdr->hdr_restart.hdr_restartseq == recvreq->req_restartseq) { - opal_output_verbose(20, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: received duplicate: dropping RNDVRESTARTNOTIFY " - "message PML:req=%d,hdr=%d CTX:req=%d,hdr=%d SRC:req=%d,hdr=%d " - "RQS:req=%d,hdr=%d src_req=%p, dst_req=%p, peer=%d", - (uint16_t)recvreq->req_msgseq, hdr->hdr_match.hdr_seq, - recvreq->req_recv.req_base.req_comm->c_contextid, hdr->hdr_match.hdr_ctx, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - hdr->hdr_match.hdr_src, recvreq->req_restartseq, - hdr->hdr_restart.hdr_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - return; - } - - /* Increment restart number. */ - recvreq->req_restartseq++; - recvreq->req_errstate |= RECVREQ_RNDVRESTART_RECVED; - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: received: outstanding receive events=%d, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - recvreq->req_events, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - - if (0 == recvreq->req_events) { - mca_pml_bfo_recv_request_rndvrestartack(recvreq, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY, - OMPI_SUCCESS, btl); - } - - return; -} - -/** - * Callback for when a RNDVRESTARTACK message is received. This - * message is sent from the receiver to the sender to acknowledge - * the receipt of the RNDVRESTARTNOTIFY message. At this point, - * the sender can reset the send request and restart the message. - */ -void mca_pml_bfo_recv_frag_callback_rndvrestartack(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_send_request_t* sendreq; - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK); - sendreq = (mca_pml_bfo_send_request_t*)hdr->hdr_restart.hdr_src_req.pval; - - /* Check to see if we have received a duplicate message. The - * first three comparisons make sure that we are not looking at a - * recycled request. The last check makes sure we are not getting - * a duplicate message for this specific request. All of this is - * needed because the receiver might get an error and repost the - * RNDVRESTARTACK message, but the RNDVRESTARTACK was actually received. */ - if ((hdr->hdr_match.hdr_ctx != sendreq->req_send.req_base.req_comm->c_contextid) || - (hdr->hdr_match.hdr_src != sendreq->req_send.req_base.req_peer) || - (hdr->hdr_match.hdr_seq != (uint16_t)sendreq->req_send.req_base.req_sequence) || - (hdr->hdr_restart.hdr_restartseq != sendreq->req_restartseq)) { - opal_output_verbose(20, mca_pml_bfo_output, - "RNDVRESTARTACK: received: does not match request, dropping " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d EXP:exp=%d,act=%d " - "src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, hdr->hdr_match.hdr_seq, - sendreq->req_send.req_base.req_comm->c_contextid, hdr->hdr_match.hdr_ctx, - sendreq->req_send.req_base.req_peer, hdr->hdr_match.hdr_src, - sendreq->req_restartseq, hdr->hdr_restart.hdr_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - return; - } - - sendreq->req_restart++; - if (2 == sendreq->req_restart) { - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTACK: received: restarting send " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - hdr->hdr_match.hdr_seq, hdr->hdr_restart.hdr_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - mca_pml_bfo_send_request_restart(sendreq, false, 0); - } else { - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTACK received: waiting for RNDVRESTARTNOTIFY completion " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - hdr->hdr_match.hdr_seq, hdr->hdr_restart.hdr_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - } - return; -} - - -/** - * Callback for when a RECVERRNOTIFY message is received. This message - * is sent from the receiver to the sender and tells the sender that - * the receiver has seen an error. This will trigger the sender - * to start the request restart sequence. - */ -void mca_pml_bfo_recv_frag_callback_recverrnotify(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_send_request_t* sendreq; - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY); - sendreq = (mca_pml_bfo_send_request_t*)hdr->hdr_restart.hdr_src_req.pval; - - /* First make sure that this message is pointing to a valid request. - * This can be determined if the communicator context, the source of - * the message, and the MPI sequence number all match. */ - if ((hdr->hdr_match.hdr_ctx != sendreq->req_send.req_base.req_comm->c_contextid) || - (hdr->hdr_match.hdr_src != sendreq->req_send.req_base.req_peer) || - (hdr->hdr_match.hdr_seq != (uint16_t)sendreq->req_send.req_base.req_sequence)) { - opal_output_verbose(20, mca_pml_bfo_output, - "RECVERRNOTIFY: received: does not match request, dropping " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d RQS:exp=%d,act=%d " - "src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, hdr->hdr_match.hdr_seq, - sendreq->req_send.req_base.req_comm->c_contextid, hdr->hdr_match.hdr_ctx, - sendreq->req_send.req_base.req_peer, hdr->hdr_match.hdr_src, - sendreq->req_restartseq, hdr->hdr_restart.hdr_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - return; - } - - /* If a good ACK was never received, then the first ACK received - * might be a RECVERRNOTIFY message. In that case, the sendreq does not - * have a valid req_recv pointer in it. Therefore, check for that - * case and update the field in the sendreq if necessary. */ - if (NULL == sendreq->req_recv.pval) { - sendreq->req_recv = hdr->hdr_restart.hdr_dst_req; - } - - /* Now check to see a restart needs to be issued. The request - * sequence number in the header is compared against the current - * request sequence number in the send request. If the header - * sequence number is greater than or equal to the send request - * number, then a rndvrestartnotify is issued. There are some cases - * where a few extra rndvrestartnotifys are issued. That is OK as - * it will all work itself out. The idea is to prevent many - * restarts unnecessarily. This still allows multiple restarts to - * happen. It could be that sometime later another error occurs - * which initiates a restart. That is OK as it will have the new - * sequence number and all is well. */ - if (hdr->hdr_restart.hdr_restartseq >= sendreq->req_restartseq) { - assert(sendreq->req_send.req_base.req_ompi.req_state == OMPI_REQUEST_ACTIVE); - sendreq->req_error++; - opal_output_verbose(30, mca_pml_bfo_output, - "RECVERRNOTIFY: received: sendreq has error, outstanding events=%d, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - sendreq->req_events, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, (void *)sendreq, - sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - - if (0 == sendreq->req_events) { - mca_pml_bfo_send_request_rndvrestartnotify(sendreq, false, - MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY, - OMPI_SUCCESS, btl); - } - } else { - opal_output_verbose(30, mca_pml_bfo_output, - "RECVERRNOTIFY: received: error has already been noted, ignoring " - "PML:exp=%d,act=%d RQS:exp=%d,act=%d src_req=%p, dst_req=%p, peer=%d", - sendreq->req_restartseq, hdr->hdr_restart.hdr_restartseq, - (uint16_t)sendreq->req_send.req_base.req_sequence, hdr->hdr_match.hdr_seq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - } - return; -} - -/** - * Callback for when a RNDVRESTARTNACK message is received. This message - * is sent from the receiver to the sender and tells the sender that - * the receiver has already completed the message and there is nothing - * else to be done. The sender should then just make the send request - * complete. - */ -void mca_pml_bfo_recv_frag_callback_rndvrestartnack(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_send_request_t* sendreq; - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK); - sendreq = (mca_pml_bfo_send_request_t*)hdr->hdr_restart.hdr_src_req.pval; - - /* Not convinced a RNDVRESTARTNACK that does not match a request can - * happen, but have the check in here anyways for now */ - if ((hdr->hdr_match.hdr_ctx != sendreq->req_send.req_base.req_comm->c_contextid) || - (hdr->hdr_match.hdr_src != sendreq->req_send.req_base.req_peer) || - (hdr->hdr_match.hdr_seq != (uint16_t)sendreq->req_send.req_base.req_sequence) || - (hdr->hdr_restart.hdr_restartseq != sendreq->req_restartseq)) { - opal_output_verbose(20, mca_pml_bfo_output, - "RNDVRESTARTNACK: received: does not match request, dropping " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d EXP:exp=%d,act=%d " - "src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, hdr->hdr_match.hdr_seq, - sendreq->req_send.req_base.req_comm->c_contextid, hdr->hdr_match.hdr_ctx, - sendreq->req_send.req_base.req_peer, hdr->hdr_match.hdr_src, - sendreq->req_restartseq, hdr->hdr_restart.hdr_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - return; - } - - opal_output_verbose(20, mca_pml_bfo_output, - "RNDVRESTARTNACK: received: marking send request as complete " - "PML=%d CTX=%d SRC=%d EXP=%d " - "src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_send.req_base.req_peer, sendreq->req_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - /* Mark the sender complete. This data exchange is over. */ - send_request_pml_complete(sendreq); - return; -} - - -/** - * This function gets called when failover is enabled and an error - * occurs during the rendezvous protocol. A message is sent to the - * receiving side notifying the request that the communication is - * going to be starting over. However, none of the information in the - * send request is reset yet, so that any in flight fragments can - * still find a home. Information in the send request gets reset when - * the completion event for this send occurs AND an ACK has been - * received back from the receiver. - */ -void mca_pml_bfo_send_request_rndvrestartnotify(mca_pml_bfo_send_request_t* sendreq, - bool repost, mca_btl_base_tag_t tag, - int status, mca_btl_base_module_t* btl) -{ - mca_btl_base_descriptor_t* des; - mca_pml_bfo_restart_hdr_t* restart; - int rc; - mca_bml_base_btl_t* bml_btl; - ompi_proc_t* proc = (ompi_proc_t*)sendreq->req_send.req_base.req_proc; - mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - /* If this message is not a repost, then update the sequence number. */ - if (!repost) { - /* Bump up the rendezvous request sequence number. */ - sendreq->req_restartseq++; - } - - assert(0 == sendreq->req_events); - assert(0 != bml_endpoint->btl_eager.arr_size); - - /* In the case that this is started because the receiver has - * sent us a message, then attempt to use a different BTL than the - * error message was received on. This may potentially tickle the - * error sooner if this side has not seen it yet. */ - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - if (bml_btl->btl == btl) { - /* If there is more than one BTL left, then we will get a - * different one. If there is only one, we will just get - * the same one back again. That is OK. */ - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - } - - /* allocate descriptor */ - mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_restart_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | - MCA_BTL_DES_SEND_ALWAYS_CALLBACK); - if( OPAL_UNLIKELY(NULL == des) ) { - opal_output(0, "%s:%d Our of resources, cannot proceed", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - - /* fill out header */ - restart = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - restart->hdr_match.hdr_common.hdr_flags = 0; - restart->hdr_match.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY; - restart->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - restart->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - restart->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - restart->hdr_restartseq = sendreq->req_restartseq; - restart->hdr_src_req.pval = sendreq; - restart->hdr_dst_req = sendreq->req_recv; - restart->hdr_dst_rank = sendreq->req_send.req_base.req_peer; /* Needed for NACKs */ - restart->hdr_jobid = OMPI_PROC_MY_NAME->jobid; - restart->hdr_vpid = OMPI_PROC_MY_NAME->vpid; - - bfo_hdr_hton(restart, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY, proc); - - /* initialize descriptor */ - des->des_cbfunc = mca_pml_bfo_rndvrestartnotify_completion; - - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: sent: PML=%d, RQS(new)=%d, CTX=%d, SRC=%d, " - "src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, sendreq->req_restartseq, - restart->hdr_match.hdr_ctx, restart->hdr_match.hdr_src, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY); - if( OPAL_UNLIKELY( rc < 0 ) ) { - opal_output(0, "[%s:%d] Cannot send rndvrestartnotify message", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - -} - -/** - * This function restarts a RNDV send request. When this is called, - * all the fields in the send request are reset and the send is - * started over. The sendreq->req_restartseq will be non-zero which will - * trigger a special flag in the RNDV header which indicates the match - * has already happened on the receiving side. - */ -void mca_pml_bfo_send_request_restart(mca_pml_bfo_send_request_t* sendreq, - bool repost, mca_btl_base_tag_t tag) -{ - size_t offset = 0; - opal_list_item_t *first_item; - opal_list_item_t *last_item; - mca_bml_base_endpoint_t* endpoint; - size_t i; - - /* If the tag is something valid, it was a repost. We could also - * check the repost field as well. Maybe I can drop the - * repost and have the tag double as it. */ - switch (tag) { - case MCA_PML_BFO_HDR_TYPE_RNDV: - opal_output_verbose(30, mca_pml_bfo_output, - "RNDV: completion failed, reset and repost: PML=%d, RQS=%d, " - "CTX=%d, SRC=%d, src_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, sendreq->req_restartseq, - sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_send.req_base.req_comm->c_my_rank, (void *)sendreq, - sendreq->req_send.req_base.req_peer); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - opal_output_verbose(30, mca_pml_bfo_output, - "RGET: completion failed, reset and repost: PML=%d, RQS=%d, " - "CTX=%d, SRC=%d, src_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, sendreq->req_restartseq, - sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_send.req_base.req_comm->c_my_rank, (void *)sendreq, - sendreq->req_send.req_base.req_peer); - break; - default: - break; - } - - /* Return mpool resources, they get reacquired when request starts over. */ - mca_pml_bfo_free_rdma_resources(sendreq); - - /* Release any memory in use if this is a buffered send */ - if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED && - sendreq->req_send.req_addr != sendreq->req_send.req_base.req_addr) { - mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); - } - - /* Clear out any unsent send ranges. Recreate the functionality - * from the get_send_range() and get_next_send_range() functions. */ - OPAL_THREAD_LOCK(&sendreq->req_send_range_lock); - first_item = opal_list_get_begin(&sendreq->req_send_ranges); - last_item = opal_list_get_last(&sendreq->req_send_ranges); - while (first_item != last_item) { - opal_list_remove_item(&sendreq->req_send_ranges, last_item); - OMPI_FREE_LIST_RETURN_MT(&mca_pml_bfo.send_ranges, (ompi_free_list_item_t *)last_item); - last_item = opal_list_get_last(&sendreq->req_send_ranges); - } - OPAL_THREAD_UNLOCK(&sendreq->req_send_range_lock); - - /* Reset the converter to the beginning. */ - opal_convertor_set_position(&sendreq->req_send.req_base.req_convertor, - &offset); - - /* Bump up internal sequence number to handle possible duplicate - * RNDV messages. In the case of reposting a RNDV message, do not - * increment the value. That way, a duplicate message can be - * detected. */ - if (!repost) { - sendreq->req_restartseq++; - } - - /* This code here is essentially the same is mca_pml_bfo_send_request_start() - * but with a few modifications since we are restarting the request, not - * starting entirely from scratch. */ - endpoint = (mca_bml_base_endpoint_t*)sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - sendreq->req_endpoint = endpoint; - sendreq->req_state = 0; - sendreq->req_lock = 0; - sendreq->req_pipeline_depth = 0; - sendreq->req_bytes_delivered = 0; - sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE; - - /* Note that we do not reset the following three items. - * They stay with their original values. - * sendreq->req_send.req_base.req_sequence - * sendreq->req_restartseq - * sendreq->req_recv.pval - */ - sendreq->req_restart = 0; /* reset in case we restart again */ - sendreq->req_error = 0; /* clear error state */ - sendreq->req_events = 0; /* clear events, probably 0 anyways */ - - MCA_PML_BASE_SEND_START( &sendreq->req_send.req_base ); - - for(i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) { - mca_bml_base_btl_t* bml_btl; - int rc; - - /* select a btl */ - bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager); - rc = mca_pml_bfo_send_request_start_btl(sendreq, bml_btl); - if(OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc)) - return; - } - add_request_to_send_pending(sendreq, MCA_PML_BFO_SEND_PENDING_START, true); -} - -/** - * This function will repost a match fragment. This function has to - * handle the case where there may not be a request associated with - * the fragment and just use the information in the fragment to - * repost the send. - */ -void mca_pml_bfo_repost_match_fragment(struct mca_btl_base_descriptor_t* des) -{ - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)des->des_cbdata; - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context; - struct mca_bml_base_endpoint_t* endpoint; - int rc; - size_t offset = 0; - - /* At this point a determination has to be made whether the - * BFO_HDR_TYPE_MATCH fragment was sent via the sendi interface or - * via the regular send interface. This is important because if it - * was sent via the sendi interface, then the request associated - * with it has already been completed and released. This can be - * determined by looking at the des->des_flags field of the - * descriptor. If the ALWAYS_CALLBACK flag is set then it is known - * that there is a valid send request associated with the fragment - * and it can be used to extricate information. If ALWAYS_CALLBACK - * is not set, then the endpoint information is in the callback - * data field and where to resend the fragment can be determined - * from the fragment. */ - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - endpoint = sendreq->req_endpoint; - opal_output_verbose(30, mca_pml_bfo_output, - "MATCH: repost: src_req=%p", - (void *)sendreq); - } else { - endpoint = des->des_cbdata; - opal_output_verbose(30, mca_pml_bfo_output, - "MATCH: repost: des=%p (sendi fragment)", - (void *)des); - } - - assert(0 != endpoint->btl_eager.arr_size); - bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager); - - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - /* Reset the converter to the beginning if the message is - * not a zero-length message. In the case of zero-length - * message, the convertor is not being used. */ - if (0 != sendreq->req_send.req_bytes_packed) { - opal_convertor_set_position(&sendreq->req_send.req_base.req_convertor, - &offset); - } - rc = mca_pml_bfo_send_request_start_btl(sendreq, bml_btl); - if (OMPI_SUCCESS == rc) { - return; - } else if (OMPI_ERR_OUT_OF_RESOURCE == rc) { - opal_output_verbose(30, mca_pml_bfo_output, - "Warning: delaying reposting of BFO_HDR_TYPE_MATCH, btls=%d", - (int)sendreq->req_endpoint->btl_eager.arr_size); - add_request_to_send_pending(sendreq, MCA_PML_BFO_SEND_PENDING_START, true); - return; - } else { - opal_output(0, "%s:%d FATAL ERROR, cannot repost BFO_HDR_TYPE_MATCH", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - } else { - /* No send request available so alloc and repost explicitly */ - mca_btl_base_descriptor_t* newdes = NULL; - mca_btl_base_segment_t* oldseg; - mca_btl_base_segment_t* newseg; - - oldseg = des->des_local; - /* The alloc routine must be called with the MCA_BTL_NO_ORDER - * flag so that the allocation routine works. The allocation - * will fill in the order flag in the descriptor. */ - mca_bml_base_alloc( bml_btl, &newdes, - MCA_BTL_NO_ORDER, - oldseg->seg_len, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - if (OPAL_UNLIKELY(NULL == newdes)) { - opal_output(0, "%s:%d FATAL ERROR, cannot repost BFO_HDR_TYPE_MATCH", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - newseg = newdes->des_local; - /* Copy over all the data that is actually sent over the wire */ - memcpy(newseg->seg_addr.pval, oldseg->seg_addr.pval, oldseg->seg_len); - newseg->seg_len = oldseg->seg_len; - - /* This call will either return OMPI_SUCCESS or OMPI_ERROR. The - * OMPI_SUCCESS only says that the send request can be freed. - * It may be that the message was queued up in the BTL. */ - rc = mca_bml_base_send(bml_btl, newdes, MCA_PML_BFO_HDR_TYPE_MATCH); - - /* Some BTLs will set the CALLBACK flag but we do not want that - * as there is no longer a request associated with this descriptor. - * Therefore, always make sure it is cleared. */ - newdes->des_flags &= ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK; - - if( OPAL_LIKELY( rc >= 0 )) { - /* Just let the normal flow of data free whatever needs - * to be freed */ - return; - } else { - opal_output(0, "%s:%d FATAL ERROR, cannot repost BFO_HDR_TYPE_MATCH", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - } - /* No need to free any descriptors. The BTLs take care of it since - * we originally allocated with MCA_BTL_DES_FLAGS_BTL_OWNERSHIP. */ -} - -/** - * Completion callback for rndvrestartnotify completion event. If the - * RNDVRESTARTACK has already been received, then reset and restart. - * Otherwise, just update the state and let the RNDVRESTARTACK trigger - * the reset and restart. - */ -void -mca_pml_bfo_rndvrestartnotify_completion(mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status) -{ - mca_pml_bfo_restart_hdr_t* restart; - mca_pml_bfo_send_request_t* sendreq; - - restart = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - sendreq = (mca_pml_bfo_send_request_t*) restart->hdr_src_req.pval; - - /* Need to resend this message in the case that it fails */ - if( OPAL_UNLIKELY((OMPI_SUCCESS != status))) { - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: completion failed: repost " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, - (void *)sendreq, sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - /* Repost the message and indicate it is a repost, not a new one. No need - * to check the req_events as this is the only possible outstanding send - * event when we have posted this message. We also know the sendreq is still - * available because nothing can proceed until this completion event happens - * successfully as we track the req_restart value. */ - mca_pml_bfo_send_request_rndvrestartnotify(sendreq, true, - MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY, - status, btl); - return; - } - - /* The req_restart value is incremented to indicate completion of - * the RNDVRESTARTNOTIFY message. Then (typically) the arrival of the - * ACK message will cause the request to reset and restart. Need to - * make sure that RNDVRESTARTNOTIFY callback has been called as well as - * the ACK back from the receiver prior to resetting and restarting - * the request. This is needed in case we get an error on the - * RNDVRESTARTNOTIFY message, but it actually makes it over. We want - * to make sure the send request has not restarted yet. So, keep a - * counter that counts to 2. */ - sendreq->req_restart++; - if (2 == sendreq->req_restart) { - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: completion: restarting request " - "PML=%d, RQS=%d, CTX=%d, src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, - sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_recv.pval, (void *)sendreq, - sendreq->req_send.req_base.req_peer); - mca_pml_bfo_send_request_restart(sendreq, false, 0); - } else { - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNOTIFY: completion: waiting for ack " - "PML=%d, RQS=%d, CTX=%d, src_req=%p, dst_req=%p, peer=%d", - (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, - sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_recv.pval, (void *)sendreq, - sendreq->req_send.req_base.req_peer); - } -} - -/** - * This function is called when an error is detected on a completion - * event on the receiving side. This can come from a ACK, PUT, RDMA - * read (GET) or RECVERRNOTIFY completion event. When this happens, check - * the state of the request and decide if the sender needs be notified - * that a problem was seen. If no RECVERRNOTIFY message has been sent and - * no RNDVRESTARTNOTIFY has been received from the sender, then send a - * message telling the sender an error was seen. - */ -void mca_pml_bfo_recv_request_recverrnotify(mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_tag_t tag, int status) -{ - mca_btl_base_descriptor_t* des; - mca_pml_bfo_restart_hdr_t* restart; - ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc; - mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - mca_bml_base_btl_t* bml_btl; - int rc; - - assert(0 != bml_endpoint->btl_eager.arr_size); - - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - - /* allocate descriptor */ - mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_restart_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | - MCA_BTL_DES_SEND_ALWAYS_CALLBACK); - if( OPAL_UNLIKELY(NULL == des) ) { - opal_output(0, "%s:%d Out of resources, cannot proceed", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - - /* fill out header */ - restart = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - restart->hdr_match.hdr_common.hdr_flags = 0; - restart->hdr_match.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY; - restart->hdr_match.hdr_ctx = recvreq->req_recv.req_base.req_comm->c_contextid; - restart->hdr_match.hdr_src = recvreq->req_recv.req_base.req_comm->c_my_rank; - restart->hdr_match.hdr_seq = (uint16_t)recvreq->req_msgseq; - restart->hdr_restartseq = recvreq->req_restartseq; - restart->hdr_src_req = recvreq->remote_req_send; - restart->hdr_dst_req.pval = recvreq; - - bfo_hdr_hton(restart, MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY, proc); - - /* initialize descriptor */ - des->des_cbfunc = mca_pml_bfo_recv_restart_completion; - - opal_output_verbose(30, mca_pml_bfo_output, - "RECVERRNOTIFY: sending to sender, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d, btl=%p", - recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, - (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - (void *)bml_btl->btl); - - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY); - if( OPAL_UNLIKELY( rc < 0 ) ) { - opal_output(0, "[%s:%d] Cannot send recverrnotify message", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - /* Prevent future error messages on this request */ - recvreq->req_errstate |= RECVREQ_RECVERRSENT; -} - -/** - * This function is called when it may be time to send a RNDVRESTARTACK - * message back to the sending side. This can happen because we - * received a RNDVRESTARTNOTIFY message from the sender. This can - * also happen if we have noticed that the request has received the - * RNDVRESTARTNOTIFY message, but has not yet sent out the RNDVRESTARTACK - * because there were still some pending receive events on the request. - * That means we can enter this routine from a completion event on a ACK, - * PUT, or RDMA read as well as from the receipt of a RNDVRESTARTNOTIFY - * message. If all is good, we sent the RNDVRESTARTACK message back to - * the sender. Then sometime later a message will arrive telling us - * to reset and restart the receive request. - */ -void mca_pml_bfo_recv_request_rndvrestartack(mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_tag_t tag, int status, - mca_btl_base_module_t* btl) -{ - mca_btl_base_descriptor_t* des; - mca_pml_bfo_restart_hdr_t* restart; - ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc; - mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - mca_bml_base_btl_t* bml_btl; - int rc; - - assert((recvreq->req_errstate & RECVREQ_RNDVRESTART_RECVED) == RECVREQ_RNDVRESTART_RECVED); - assert((recvreq->req_errstate & RECVREQ_RNDVRESTART_ACKED) == 0); - assert(0 != bml_endpoint->btl_eager.arr_size); - - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - - /* Attempt to use a different BTL than the error message was - * received on. This may potentially tickle the error sooner if - * this side has not seen it yet. */ - if (bml_btl->btl == btl) { - /* If there is more than one BTL left, then we will get a - * different one. If there is only one, we will just get - * the same one back again. That is OK. */ - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - } - - /* allocate descriptor */ - mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_restart_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | - MCA_BTL_DES_SEND_ALWAYS_CALLBACK); - if( OPAL_UNLIKELY(NULL == des) ) { - opal_output(0, "%s:%d Out of resources, cannot proceed", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - - /* fill out header */ - restart = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - restart->hdr_match.hdr_common.hdr_flags = 0; - restart->hdr_match.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK; - restart->hdr_match.hdr_ctx = recvreq->req_recv.req_base.req_comm->c_contextid; - restart->hdr_match.hdr_src = recvreq->req_recv.req_base.req_comm->c_my_rank; - restart->hdr_match.hdr_seq = (uint16_t)recvreq->req_msgseq; - restart->hdr_restartseq = recvreq->req_restartseq; - restart->hdr_src_req = recvreq->remote_req_send; - restart->hdr_dst_req.pval = recvreq; - - bfo_hdr_hton(restart, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK, proc); - - /* initialize descriptor */ - des->des_cbfunc = mca_pml_bfo_recv_restart_completion; - des->des_cbdata = (void *)proc; - - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTACK: due to PML tag=%d completion, sending to " - "sender, PML=%d, RQS=%d, src_req=%p, dst_req=%p, status=%d, " - "peer=%d, btl=%p", - tag, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, status, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, - (void *)bml_btl->btl); - - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK); - if( OPAL_UNLIKELY( rc < 0 ) ) { - opal_output(0, "[%s:%d] Cannot send rndvrestartack message", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - /* Move to the next state so we do not send anymore ACKs */ - recvreq->req_errstate &= ~RECVREQ_RNDVRESTART_RECVED; - recvreq->req_errstate |= RECVREQ_RNDVRESTART_ACKED; -} - -/** - * Called after the receipt of a RNDVRESTARTNOTIFY message to a request - * that no longer matches. This can happen if the sender detected an - * error, but the receiver actually received all the data. Therefore - * send a NACK back instead of the ACK so that the sender can complete - * its request. This happens very rarely. Note that we need to make - * use of the hdr_dst_rank that we received from the notify message. - * This is so the sending side make sure the message matches a valid - * request on the sending side. - */ -void mca_pml_bfo_recv_request_rndvrestartnack(mca_btl_base_descriptor_t* olddes, - ompi_proc_t* ompi_proc, bool repost) -{ - mca_btl_base_segment_t* segments; - mca_pml_bfo_restart_hdr_t* hdr; /* hdr of NOTIFY message */ - mca_pml_bfo_restart_hdr_t* nack; /* hdr of NACK message */ - mca_btl_base_descriptor_t* des; - mca_bml_base_endpoint_t* bml_endpoint; - mca_bml_base_btl_t* bml_btl; - int rc; - - if (repost) { - /* In the case where we are reposting the NACK, the information - * is in the src area, since we are reposting a send. In addition, - * we get the ompi_proc from the old descriptor. */ - ompi_proc = olddes->des_cbdata; - } - - segments = olddes->des_local; - hdr = (mca_pml_bfo_restart_hdr_t*)segments->seg_addr.pval; - - bml_endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - assert(0 != bml_endpoint->btl_eager.arr_size); - bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - - /* allocate descriptor */ - mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_restart_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | - MCA_BTL_DES_SEND_ALWAYS_CALLBACK); - if( OPAL_UNLIKELY(NULL == des) ) { - opal_output(0, "%s:%d Out of resources, cannot proceed", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - - /* fill out header */ - nack = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - nack->hdr_match.hdr_common.hdr_flags = 0; - nack->hdr_match.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK; - nack->hdr_match.hdr_ctx = hdr->hdr_match.hdr_ctx; - nack->hdr_match.hdr_src = hdr->hdr_dst_rank; /* Receiver rank */ - nack->hdr_match.hdr_seq = hdr->hdr_match.hdr_seq; - nack->hdr_restartseq = hdr->hdr_restartseq; - nack->hdr_src_req = hdr->hdr_src_req; - nack->hdr_dst_req.pval = 0; - - bfo_hdr_hton(nack, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK, ompi_proc); - - /* Initialize descriptor. Save away ompi_proc in case we need - * to respost this fragmnet. */ - des->des_cbfunc = mca_pml_bfo_recv_restart_completion; - des->des_cbdata = ompi_proc; - - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNACK: sending to sender, " - "PML=%d, RQS=%d, CTX=%d, SRC=%d, peer=%d", - nack->hdr_match.hdr_seq, nack->hdr_restartseq, - nack->hdr_match.hdr_ctx, nack->hdr_match.hdr_src, - OMPI_CAST_RTE_NAME(&ompi_proc->super.proc_name)->vpid); - - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK); - if( OPAL_UNLIKELY( rc < 0 ) ) { - opal_output(0, "[%s:%d] Cannot send rndvrestartnack message", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } -} - - -/** - * Reset all the receive request fields to match what a request - * looks like when it is first started. This gets called when - * the rendezvous/rget message is being restarted. - */ -void mca_pml_bfo_recv_request_reset(mca_pml_bfo_recv_request_t* match) { - int i; - - assert(true != match->req_recv.req_base.req_pml_complete); - - /* Free up any resources that were reserved for this receive. This - * was copied from the receive completion code. */ - for(i = 0; i < (int)match->req_rdma_cnt; i++) { - mca_mpool_base_registration_t* btl_reg = match->req_rdma[i].btl_reg; - if( NULL != btl_reg && btl_reg->mpool != NULL) { - btl_reg->mpool->mpool_deregister( btl_reg->mpool, btl_reg ); - } - } - match->req_rdma_cnt = 0; - - /* This code is mostly copied from mca_pml_bfo_recv_req_start. - * Note 1: Leave req_bytes_expected as the original value. No - * need to adjust this as it is set when convertor is created. - * Note 2: Leave req_bytes_delivered as the original value. - * This is created when the convertor is created and represents - * the expected bytes from the user. */ - assert(0 == match->req_events); - match->req_errstate = 0; - match->req_lock = 0; - match->req_pipeline_depth = 0; - match->req_bytes_received = 0; - match->req_rdma_idx = 0; - match->req_rdma_offset = 0; - match->req_send_offset = 0; - match->req_pending = false; - match->req_ack_sent = false; - match->req_restartseq++; - - /* These really should not need to be set, but this matches some - * of the initialization within MCA_PML_BASE_RECV_START. */ - match->req_recv.req_base.req_pml_complete = false; - match->req_recv.req_base.req_ompi.req_complete = false; - match->req_recv.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE; - - /* Reset the convertor */ - opal_convertor_set_position(&match->req_recv.req_base.req_convertor, - &match->req_rdma_offset); - return; -} - -/* - * Completion callback for RNDVRESTARTACK, RNDVRESTARTNACK and RECVERRNOTIFY. - */ -void mca_pml_bfo_recv_restart_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - if(OPAL_UNLIKELY(OMPI_SUCCESS != status)) { - mca_pml_bfo_common_hdr_t* common = des->des_local->seg_addr.pval; - mca_pml_bfo_restart_hdr_t* restart; /* RESTART header */ - mca_pml_bfo_recv_request_t* recvreq; - - switch (common->hdr_type) { - case MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK: - restart = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - recvreq = (mca_pml_bfo_recv_request_t*) restart->hdr_dst_req.pval; - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTACK: completion failed: try again " - "PML:req=%d,hdr=%d RQS:req=%d,hdr=%d CTX:req=%d,hdr=%d " - "src_req=%p, dst_req=%p, peer=%d", - recvreq->req_msgseq, restart->hdr_match.hdr_seq, - recvreq->req_restartseq, restart->hdr_restartseq, - recvreq->req_recv.req_base.req_comm->c_contextid, - restart->hdr_match.hdr_ctx, - recvreq->remote_req_send.pval, - (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - - /* Adjust the states back to avoid assert errors */ - recvreq->req_errstate &= ~RECVREQ_RNDVRESTART_ACKED; - recvreq->req_errstate |= RECVREQ_RNDVRESTART_RECVED; - mca_pml_bfo_recv_request_rndvrestartack(recvreq, MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK, - status, btl); - break; - case MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK: - opal_output_verbose(30, mca_pml_bfo_output, - "RNDVRESTARTNACK: completion failed: try again " - "des=%p ", (void *)des); - /* Just blast it again. No request associated with it. */ - mca_pml_bfo_recv_request_rndvrestartnack(des, NULL, true); - break; - case MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY: - restart = (mca_pml_bfo_restart_hdr_t*)des->des_local->seg_addr.pval; - recvreq = (mca_pml_bfo_recv_request_t*) restart->hdr_dst_req.pval; - /* With just two BTLs, this should never happen as we are - * typically sending the RECVERRNOTIFY message on the - * working BTL. But, just in case, if we get an error, - * send it again. */ - opal_output_verbose(30, mca_pml_bfo_output, - "RECVERRNOTIFY: completion failed: try again, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, - (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - mca_pml_bfo_recv_request_recverrnotify(recvreq, MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY, - status); - break; - default: - opal_output(0, "[%s:%d] Unknown callback error", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - } -} - -/* - * Remove a btl for future communication on an endpoint. - */ -void mca_pml_bfo_map_out_btl(struct mca_btl_base_module_t* btl, - ompi_proc_t *errproc, char *btlname) -{ - mca_bml_base_endpoint_t* ep; - bool remove = false; - int i; - - ep = (mca_bml_base_endpoint_t*)errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - /* The bml_del_proc_btl function does not indicate if it - * actually removed a btl, so let me check up front. This is - * done so that we can only print out messages when a btl is - * actually going to be removed. These arrays are small so it - * is OK to walk through all of them even though it may be - * redundant. */ - for( i = 0; i < (int)ep->btl_eager.arr_size; i++ ) { - if( ep->btl_eager.bml_btls[i].btl == btl ) { - remove = true; - } - } - for( i = 0; i < (int)ep->btl_send.arr_size; i++ ) { - if( ep->btl_send.bml_btls[i].btl == btl ) { - remove = true; - } - } - for( i = 0; i < (int)ep->btl_rdma.arr_size; i++ ) { - if( ep->btl_rdma.bml_btls[i].btl == btl ) { - remove = true; - } - } - - if (true == remove) { - mca_bml.bml_del_proc_btl(errproc, btl); - - opal_output_verbose(10, mca_pml_bfo_output, - "BTL %s error: rank=%d mapping out %s " - "to rank=%d on node=%s \n", - btl->btl_component->btl_version.mca_component_name, - OMPI_PROC_MY_NAME->vpid, - btlname, OMPI_CAST_RTE_NAME(&errproc->super.proc_name)->vpid, - (NULL == errproc->super.proc_hostname) ? "unknown" : errproc->super.proc_hostname); - - /* Need to search for any pending packets associated - * with this endpoint and remove them. We may also - * have to restarts depending on the state of the - * requests. */ - mca_pml_bfo_error_pending_packets(btl, ep); - - if ((ep->btl_eager.arr_size == 0) && - (ep->btl_send.arr_size == 0) && - (ep->btl_rdma.arr_size == 0)) { - opal_output(0, "%s:%d: No more interfaces, aborting", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - } -} - -void mca_pml_bfo_failover_error_handler(struct mca_btl_base_module_t* btl, - int32_t flags, ompi_proc_t *errproc, char *btlname) -{ - ompi_proc_t** procs; - size_t p, num_procs; - - /* If we are in here, we know that the we were called - * with the flags == MCA_BTL_ERROR_FLAGS_NONFATAL so no - * need to check it in here. */ - assert(flags & MCA_BTL_ERROR_FLAGS_NONFATAL); - - procs = ompi_proc_all(&num_procs); - - if(NULL == procs) { - opal_output(0, "%s:%d: Out of memory, giving up.", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - - if (NULL == btlname) { - btlname = "unknown"; - } - - /* If the process to map out is not specified then map out the - * entire BTL. Otherwise, only map out the BTL for the specific - * remote process. */ - if (NULL == errproc) { - for( p = 0; p < num_procs; p++ ) { - mca_pml_bfo_map_out_btl(btl, procs[p], btlname); - } - } else { - mca_pml_bfo_map_out_btl(btl, errproc, btlname); - } - free(procs); -} - -/** - * This function is called since when we are mapping out a BML. This - * will walk through the four PML lists and dispatch with the - * fragments/requests. There are four different lists and each one is - * handled slighty differently. In all cases, we first see if the - * message is associated with the endpoint that is being mapped out. - * If not, then just leave it alone and put it back on the list. If - * it is associated with the endpoint, then a each list handles it - * slighlty differently. Also, in some cases, we actually adjust the - * pointers to the BMLs in the messages as they may have changed when - * the BML is mapped out. That is because this is called after we - * have mapped out the offending BML and adjusted the array of - * available BMLs. - */ -static void mca_pml_bfo_error_pending_packets(mca_btl_base_module_t* btl, - mca_bml_base_endpoint_t* ep) { - int32_t i, s; - - /* The pckt_pending list contains both ACK and FIN messages. - * ACKs can be sent over any BTL associated with the endpoint. - * Therefore, the bml_btl entry for ACKS is NULL and they do - * not need to be adjusted. It is also worth noting that - * the ACK will be the only outstanding message associated - * with a request so we can just let nature takes it course. - * - * FIN messages do have a BML associated with them, but they - * can also be sent over any BTL. Therefore, adjust the bml - * pointer in the pckt to ensure it points at a valid BML. - */ - - s = (int32_t)opal_list_get_size(&mca_pml_bfo.pckt_pending); - for(i = 0; i < s; i++) { - mca_pml_bfo_pckt_pending_t *pckt; - opal_output_verbose(0, mca_pml_bfo_output, - "INFO: pckt_pending list has %d entries", s); -#if 1 - /* TODO: Error out until code is tested */ - opal_output_verbose(0, mca_pml_bfo_output, - "%s:%d: Support not implemented, aborting", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - pckt = (mca_pml_bfo_pckt_pending_t*) - opal_list_remove_first(&mca_pml_bfo.pckt_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - - /* My guess is that this can happen in the threaded - * case where the other thread removed some packets - * after we determined the size of the list. */ - if(NULL == pckt) - break; - - /* If there is no bml stored on the packet, then just - * put it back on the list as there is nothing to adjust. - * This appears to be true with ACK packets. */ - if (NULL == pckt->bml_btl) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.pckt_pending, - (opal_list_item_t*)pckt); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - continue; - } - - /* Now see if this endpoint matches the one we are mapping - * out. If so, adjust the bml entry so to ensure it is - * not pointing at a stale bml. We do not really care - * which BML it is pointing at as long as it is valid. - * In either case, then put entry back on the list. */ - if (pckt->proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] == ep) { - opal_output_verbose(15, mca_pml_bfo_output, - "INFO: Found matching pckt on pckt_pending list, adjusting bml"); - pckt->bml_btl = mca_bml_base_btl_array_get_next(&ep->btl_eager); - } - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.pckt_pending, - (opal_list_item_t*)pckt); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - - } - - /* This next list holds rdma fragments. We need to walk through - * the list and see if any are associated with the endpoint - * we are mapping out. If not, then just put back on the - * list. If they are, then we need to error them out. One issue - * is that we need to deal with the case where there may be more - * then one pending rdma fragment for a request. */ - s = (int32_t)opal_list_get_size(&mca_pml_bfo.rdma_pending); - for(i = 0; i < s; i++) { - mca_pml_bfo_rdma_frag_t* frag; - mca_pml_bfo_send_request_t* sendreq; - mca_pml_bfo_recv_request_t* recvreq; - opal_output_verbose(0, mca_pml_bfo_output, - "INFO: rdma_pending list has %d entries", s); -#if 1 - /* TODO: Error out until code is tested */ - opal_output_verbose(0, mca_pml_bfo_output, - "%s:%d: Support not implemented, aborting", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - frag = (mca_pml_bfo_rdma_frag_t*) - opal_list_remove_first(&mca_pml_bfo.rdma_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - - /* My guess is that this can happen in the threaded - * case where the other thread removed some packets - * after we determined the size of the list. */ - if(NULL == frag) - break; - - /* Check to see if it matches our endpoint. If it does, - * then check if it matches the BTL that is being mapped - * out. If it does not, then just readjust the BML pointer. - * If it does, then we need to do something with it. */ - if (frag->rdma_ep != ep) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.rdma_pending, - (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - continue; - } - - /* If we are here, then we know we are working on the same - * endpoint. Now check the BTL. */ - if (frag->rdma_btl != btl) { - opal_output_verbose(15, mca_pml_bfo_output, - "INFO: Found matching frag on rdma_pending list, adjusting bml"); - /* The BTL this RDMA is associated with is not the - * one that is getting mapped out, so just adjust the - * BML pointer and put back on the list. */ - frag->rdma_bml = mca_bml_base_btl_array_find(&ep->btl_rdma, frag->rdma_btl); - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.rdma_pending, - (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - continue; - } - - /* Now we call the restart routine. This is just like if we got - * a completion event after calling an RDMA write. This will - * take care of figuring out if we need to restart the request - * or wait for any outstanding events to complete. */ - if(frag->rdma_state == MCA_PML_BFO_RDMA_PUT) { - opal_output_verbose(15, mca_pml_bfo_output, - "INFO: Found matching PUT frag on rdma_pending list, restarting"); - sendreq = frag->rdma_req; - mca_pml_bfo_send_request_rndvrestartnotify(sendreq, false, - MCA_PML_BFO_HDR_TYPE_PUT, 2, btl); - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); - } else { - opal_output_verbose(15, mca_pml_bfo_output, - "INFO: Found matching RGET frag on rdma_pending list, sending reqerror"); - /* This is just like what we do on an rget completion event */ - recvreq = (mca_pml_bfo_recv_request_t*)frag->rdma_req; - mca_pml_bfo_recv_request_recverrnotify(recvreq, MCA_PML_BFO_HDR_TYPE_RGET, 2); - - /* See if the request has received a RNDVRESTARTNOTIFY */ - if( OPAL_UNLIKELY(recvreq->req_errstate)) { - if (recvreq->req_errstate & RECVREQ_RNDVRESTART_RECVED) { - mca_pml_bfo_recv_request_rndvrestartack(recvreq, - MCA_PML_BFO_HDR_TYPE_RGET, - 2, btl); - } - } - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); - } - } - - s = opal_list_get_size(&mca_pml_bfo.send_pending); - /* Look for pending events on our endpoint */ - for(i = 0; i < s; i++) { - mca_pml_bfo_send_request_t* sendreq; - ompi_proc_t* proc; - mca_bml_base_endpoint_t* bml_endpoint; - opal_output_verbose(0, mca_pml_bfo_output, - "INFO: send_pending list has %d entries", s); -#if 1 - /* TODO: Error out until code is tested */ - opal_output_verbose(0, mca_pml_bfo_output, - "%s:%d: Support not implemented, aborting", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - sendreq = (mca_pml_bfo_send_request_t*) - opal_list_remove_first(&mca_pml_bfo.send_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - - /* My guess is that this can happen in the threaded - * case where the other thread removed some packets - * after we determined the size of the list. */ - if(NULL == sendreq) - break; - - proc = (ompi_proc_t*)sendreq->req_send.req_base.req_proc; - bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - /* Check to see if it matches our endpoint. If it does not, - * then just put it back on the list as there is nothing - * we need to do with it. */ - if (bml_endpoint != ep) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.send_pending, - (opal_list_item_t*)sendreq); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - continue; - } - - switch(sendreq->req_pending) { - case MCA_PML_BFO_SEND_PENDING_SCHEDULE: - /* If this send request is using the endpoint that received - * the error, then let us error it out. In the case - * where there is only one fragment left to be scheduled - * and it would have gone over the good BTL, this is - * not necessary. But, we will use simplicity here - * and assume that some of the fragments are still - * scheduled to go over the broken BTL. */ - sendreq->req_error++; - mca_pml_bfo_send_request_rndvrestartnotify(sendreq, false, - MCA_PML_BFO_HDR_TYPE_FRAG, 2, btl); - break; - case MCA_PML_BFO_SEND_PENDING_START: - /* If the request has not even started, then just put it back - * on the list. Nothing else to do with it. */ - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.send_pending, - (opal_list_item_t*)sendreq); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - break; - default: - opal_output(0, "[%s:%d] wrong send request type\n", - __FILE__, __LINE__); - break; - } - } - - s = (int)opal_list_get_size(&mca_pml_bfo.recv_pending); - for(i = 0; i < s; i++) { - mca_pml_bfo_recv_request_t* recvreq; - ompi_proc_t* proc; - mca_bml_base_endpoint_t* bml_endpoint; - opal_output_verbose(0, mca_pml_bfo_output, - "INFO: recv_pending list has %d entries", s); -#if 1 - /* TODO: Error out until code is tested */ - opal_output_verbose(0, mca_pml_bfo_output, - "%s:%d: Support not implemented, aborting", - __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - recvreq = (mca_pml_bfo_recv_request_t*) - opal_list_remove_first(&mca_pml_bfo.recv_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - - /* My guess is that this can happen in the threaded - * case where the other thread removed some packets - * after we determined the size of the list. */ - if(NULL == recvreq) - break; - - proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc; - bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - if (bml_endpoint != ep) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.recv_pending, - (opal_list_item_t*)recvreq); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - continue; - } - - mca_pml_bfo_recv_request_recverrnotify(recvreq, MCA_PML_BFO_HDR_TYPE_PUT, 2); - } -} - -/** - * Call each time we get a completion event on ACK or PUT message. - * These types of messages are receive control type messages. This - * function is only called if the underlying BTL supports failover. - * Otherwise, there is no need for this check. - */ -void mca_pml_bfo_check_recv_ctl_completion_status(mca_btl_base_module_t* btl, - struct mca_btl_base_descriptor_t* des, - int status) -{ - mca_pml_bfo_common_hdr_t * common = des->des_local->seg_addr.pval; - mca_pml_bfo_rdma_hdr_t* hdr; /* PUT header */ - struct mca_btl_base_descriptor_t* rdma_des; - mca_pml_bfo_recv_request_t* recvreq; - - if(OPAL_UNLIKELY(OMPI_SUCCESS != status)) { - switch (common->hdr_type) { - case MCA_PML_BFO_HDR_TYPE_ACK: - recvreq = des->des_cbdata; - - /* Record the error. Send RECVERRNOTIFY if necessary. */ - if (recvreq->req_errstate) { - opal_output_verbose(30, mca_pml_bfo_output, - "ACK: completion failed, error already seen, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - } else { - opal_output_verbose(30, mca_pml_bfo_output, - "ACK: completion failed, sending RECVERRNOTIFY to sender, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - mca_pml_bfo_recv_request_recverrnotify(recvreq, MCA_PML_BFO_HDR_TYPE_ACK, status); - } - break; - - case MCA_PML_BFO_HDR_TYPE_PUT: - hdr = (mca_pml_bfo_rdma_hdr_t*)des->des_local->seg_addr.pval; - rdma_des = hdr->hdr_des.pval; - recvreq = des->des_cbdata; - if ((NULL != rdma_des->des_cbdata) && (recvreq == rdma_des->des_cbdata)) { - /* We now record the error, send the RECVERRNOTIFY if - * necessary, and free the descriptor. Prior to this, - * we want to ensure that we have not reached the case - * where the PUT message actually made it over and we - * have already received a FIN back. We first check to - * see if the RDMA descriptor cbdata is pointing to - * NULL. If it is, this means that the PUT message must - * have made it over and a corresponding FIN already - * made it back and freed the RDMA descriptor. Second, - * if it is non-null, we make sure that it is pointing - * to the same request as the PUT descriptor is. If - * it is not, again we assume that the FIN came back - * and freed it. And we can count on the fact that the - * recvreq has not been freed or reused as it is held - * until this very completion event occurs. */ - if (recvreq->req_errstate) { - opal_output_verbose(30, mca_pml_bfo_output, - "PUT: completion failed, error already seen, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - } else { - opal_output_verbose(30, mca_pml_bfo_output, - "PUT: completion failed, sending RECVERRNOTIFY to sender, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - mca_pml_bfo_recv_request_recverrnotify(recvreq, MCA_PML_BFO_HDR_TYPE_PUT, status); - } -#if 0 - /* TODO: Add descriptor to receive request so it can - * be freed only when receive request is freed and - * only if needed. */ - btl->btl_free(btl, rdma_des); -#endif - } - break; - default: - ompi_rte_abort(-1, NULL); - } - } - - switch (common->hdr_type) { - case MCA_PML_BFO_HDR_TYPE_ACK: - recvreq = des->des_cbdata; - recvreq->req_events--; - assert(recvreq->req_events >= 0); - if(OPAL_UNLIKELY (recvreq->req_errstate & RECVREQ_RNDVRESTART_RECVED)) { - opal_output_verbose(30, mca_pml_bfo_output, - "ACK: completion: recvreq in error, outstanding events=%d " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, status=%d, peer=%d", - recvreq->req_events, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, status, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - if (0 == recvreq->req_events) { - mca_pml_bfo_recv_request_rndvrestartack(recvreq, MCA_PML_BFO_HDR_TYPE_ACK, - status, btl); - } - return; - } - recv_request_pml_complete_check(recvreq); - break; - case MCA_PML_BFO_HDR_TYPE_PUT: - recvreq = des->des_cbdata; - recvreq->req_events--; - assert(recvreq->req_events >= 0); - if(OPAL_UNLIKELY(recvreq->req_errstate & RECVREQ_RNDVRESTART_RECVED)) { - opal_output_verbose(30, mca_pml_bfo_output, - "PUT: completion: recvreq in error, outstanding events=%d " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, status=%d, peer=%d", - recvreq->req_events, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, status, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - if (0 == recvreq->req_events) { - mca_pml_bfo_recv_request_rndvrestartack(recvreq, MCA_PML_BFO_HDR_TYPE_PUT, - status, btl); - } - return; - } - recv_request_pml_complete_check(recvreq); - break; - } -} - -/** - * Register four functions to handle extra PML message types that - * are utilized when a failover occurs. - */ -int mca_pml_bfo_register_callbacks(void) { - int rc; - /* The following four functions are utilized when failover - * support for openib is enabled. */ - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY, - mca_pml_bfo_recv_frag_callback_rndvrestartnotify, - NULL ); - if(OMPI_SUCCESS != rc) - return rc; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK, - mca_pml_bfo_recv_frag_callback_rndvrestartack, - NULL ); - if(OMPI_SUCCESS != rc) - return rc; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK, - mca_pml_bfo_recv_frag_callback_rndvrestartnack, - NULL ); - if(OMPI_SUCCESS != rc) - return rc; - - rc = mca_bml.bml_register( MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY, - mca_pml_bfo_recv_frag_callback_recverrnotify, - NULL ); - if(OMPI_SUCCESS != rc) - return rc; - - return rc; -} - -/** - * Update a few fields when we are restarting either a RNDV or - * RGET type message. - */ -void mca_pml_bfo_update_rndv_fields(mca_pml_bfo_hdr_t* hdr, - mca_pml_bfo_send_request_t* sendreq, char *type) -{ - hdr->hdr_common.hdr_flags |= MCA_PML_BFO_HDR_FLAGS_RESTART; - hdr->hdr_rndv.hdr_dst_req = sendreq->req_recv; - hdr->hdr_rndv.hdr_restartseq = sendreq->req_restartseq; - opal_output_verbose(30, mca_pml_bfo_output, - "%s: restarting: PML=%d, RQS=%d, CTX=%d, SRC=%d, " - "src_req=%p, dst_req=%p, peer=%d", - type, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, - sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_send.req_base.req_comm->c_my_rank, (void *)sendreq, - sendreq->req_recv.pval, sendreq->req_send.req_base.req_peer); -} - -/** - * The following set of functions are all called when it is determined - * that the cached bml_btl->btl does not match the btl handed back - * by the callback function. This means that the bml_btl array has - * been shuffled and the bml_btl matching the btl has to be found - * back. If it cannot be found, then just find a different one to - * use. - */ -void mca_pml_bfo_update_eager_bml_btl_recv_ctl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - struct mca_btl_base_descriptor_t* des) -{ - if ((*bml_btl)->btl != btl) { - mca_pml_bfo_common_hdr_t * common = des->des_local->seg_addr.pval; - mca_pml_bfo_ack_hdr_t* ack; /* ACK header */ - mca_pml_bfo_recv_request_t* recvreq = NULL; - char *type = NULL; - - switch (common->hdr_type) { - case MCA_PML_BFO_HDR_TYPE_ACK: - ack = (mca_pml_bfo_ack_hdr_t*)des->des_local->seg_addr.pval; - recvreq = (mca_pml_bfo_recv_request_t*) ack->hdr_dst_req.pval; - type = "ACK"; - break; - case MCA_PML_BFO_HDR_TYPE_PUT: - recvreq = des->des_cbdata; - type = "PUT"; - break; - default: - /* In theory, this can never happen. */ - opal_output(0, "%s:%d FATAL ERROR, unknown header (hdr=%d)", - __FILE__, __LINE__, common->hdr_type); - ompi_rte_abort(-1, NULL); - } - - mca_pml_bfo_find_recvreq_eager_bml_btl(bml_btl, btl, recvreq, type); - } -} - -void mca_pml_bfo_find_sendreq_eager_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_send_request_t* sendreq, - char* type) -{ - if ((*bml_btl)->btl != btl) { - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML does not match BTL, find it back, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, (void *)sendreq, - sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - *bml_btl = mca_bml_base_btl_array_find(&sendreq->req_endpoint->btl_eager, btl); - if (NULL == *bml_btl) { - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML is gone, find another one, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, (void *)sendreq, - sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - *bml_btl = mca_bml_base_btl_array_get_next(&sendreq->req_endpoint->btl_eager); - } - } -} - -void mca_pml_bfo_find_sendreq_rdma_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_send_request_t* sendreq, - char* type) -{ - if ((*bml_btl)->btl != btl) { - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML does not match BTL, find it back, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, (void *)sendreq, - sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - *bml_btl = mca_bml_base_btl_array_find(&sendreq->req_endpoint->btl_rdma, btl); - if (NULL == *bml_btl) { - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML is gone, find another one, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, (void *)sendreq, - sendreq->req_recv.pval, - sendreq->req_send.req_base.req_peer); - *bml_btl = mca_bml_base_btl_array_get_next(&sendreq->req_endpoint->btl_rdma); - } - } -} - -void mca_pml_bfo_find_recvreq_eager_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_recv_request_t* recvreq, - char* type) -{ - if ((*bml_btl)->btl != btl) { - ompi_proc_t *proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc; - mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML does not match BTL, find it back, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - - *bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_eager, btl); - if (NULL == *bml_btl) { - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML is gone, find another one, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - - *bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_eager); - } - } -} - -void mca_pml_bfo_find_recvreq_rdma_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_recv_request_t* recvreq, - char* type) -{ - if ((*bml_btl)->btl != btl) { - ompi_proc_t *proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc; - mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML does not match BTL, find it back, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - - *bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl); - if (NULL == *bml_btl) { - opal_output_verbose(25, mca_pml_bfo_output, - "%s completion: BML is gone, find another one, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", - type, recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->remote_req_send.pval, (void *)recvreq, - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); - - *bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_rdma); - } - } -} - -/** - * The completion event for the RNDV message has returned with an - * error. We know that the send request we are looking at is valid - * because it cannot be completed until the sendreq->req_state value - * reaches 0. And for the sendreq->req_state to reach 0, the - * completion event on the RNDV message must occur. So, we do not - * bother checking whether the send request is valid, because we know - * it is, but we put a few asserts in for good measure. We then check - * a few fields in the request to decide what to do. If the - * sendreq->req_error is set, that means that something has happend - * already to the request and we do not want to restart it. - * Presumably, we may have received a RECVERRNOTIFY message from the - * receiver. We also check the sendreq->req_acked field to see if it - * has been acked. If it has, then again we do not restart everything - * because obviously the RNDV message has made it to the other side. - */ -bool mca_pml_bfo_rndv_completion_status_error(struct mca_btl_base_descriptor_t* des, - mca_pml_bfo_send_request_t* sendreq) -{ - assert(((mca_pml_bfo_hdr_t*)((des)->des_local->seg_addr.pval))->hdr_match.hdr_ctx == - (sendreq)->req_send.req_base.req_comm->c_contextid); - assert(((mca_pml_bfo_hdr_t*)((des)->des_local->seg_addr.pval))->hdr_match.hdr_src == - (sendreq)->req_send.req_base.req_comm->c_my_rank); - assert(((mca_pml_bfo_hdr_t*)((des)->des_local->seg_addr.pval))->hdr_match.hdr_seq == - (uint16_t)(sendreq)->req_send.req_base.req_sequence); - if ((!(sendreq)->req_error) && (NULL == (sendreq)->req_recv.pval)) { - (sendreq)->req_events--; - /* Assume RNDV did not make it, so restart from the beginning. */ - mca_pml_bfo_send_request_restart(sendreq, true, MCA_PML_BFO_HDR_TYPE_RNDV); - return true; - } - return false; -} - -/** - * Check to see if an error has occurred on this send request. If it has - * and there are no outstanding events, then we can start the restart dance. - */ -void mca_pml_bfo_completion_sendreq_has_error(mca_pml_bfo_send_request_t* sendreq, - int status, - mca_btl_base_module_t* btl, - int type, - char *description) -{ - opal_output_verbose(30, mca_pml_bfo_output, - "%s: completion: sendreq has error, outstanding events=%d, " - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, status=%d, peer=%d", - description, - sendreq->req_events, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, (void *)sendreq, - sendreq->req_recv.pval, - status, sendreq->req_send.req_base.req_peer); - if (0 == sendreq->req_events) { - mca_pml_bfo_send_request_rndvrestartnotify(sendreq, false, - type, status, btl); - } -} - -/* If we get an error on the RGET message, then first make sure that - * header matches the send request that we are pointing to. This is - * necessary, because even though the sending side got an error, the - * RGET may have made it to the receiving side and the message transfer - * may have completed. This would then mean the send request has been - * completed and perhaps in use by another communication. So there is - * no need to restart this request. Therefore, ensure that we are - * looking at the same request that the header thinks we are looking - * at. If not, then there is nothing else to be done. */ -void mca_pml_bfo_send_ctl_completion_status_error(struct mca_btl_base_descriptor_t* des) -{ - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)des->des_cbdata; - mca_pml_bfo_hdr_t* hdr = des->des_local->seg_addr.pval; - switch (hdr->hdr_common.hdr_type) { - case MCA_PML_BFO_HDR_TYPE_RGET: - if ((hdr->hdr_match.hdr_ctx != sendreq->req_send.req_base.req_comm->c_contextid) || - (hdr->hdr_match.hdr_src != sendreq->req_send.req_base.req_comm->c_my_rank) || - (hdr->hdr_match.hdr_seq != (uint16_t)sendreq->req_send.req_base.req_sequence)) { - opal_output_verbose(30, mca_pml_bfo_output, - "RGET: completion event: dropping because no valid request " - "PML:exp=%d,act=%d CTX:exp=%d,act=%d SRC:exp=%d,act=%d " - "RQS:exp=%d,act=%d, dst_req=%p", - (uint16_t)sendreq->req_send.req_base.req_sequence, - hdr->hdr_match.hdr_seq, - sendreq->req_send.req_base.req_comm->c_contextid, - hdr->hdr_match.hdr_ctx, - sendreq->req_send.req_base.req_comm->c_my_rank, - hdr->hdr_match.hdr_src, - sendreq->req_restartseq, hdr->hdr_rndv.hdr_restartseq, - (void *)sendreq); - return; - } - mca_pml_bfo_send_request_restart(sendreq, true, MCA_PML_BFO_HDR_TYPE_RGET); - return; - default: - opal_output(0, "%s:%d FATAL ERROR, unknown header (hdr=%d)", - __FILE__, __LINE__, hdr->hdr_common.hdr_type); - ompi_rte_abort(-1, NULL); - } -} diff --git a/ompi/mca/pml/bfo/pml_bfo_failover.h b/ompi/mca/pml/bfo/pml_bfo_failover.h deleted file mode 100644 index ea4f70fdc4..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_failover.h +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/** - * @file - * Functions that implement failover capabilities. - */ - -#ifndef MCA_PML_BFO_FAILOVER_H -#define MCA_PML_BFO_FAILOVER_H - -#include "opal/mca/btl/btl.h" -#include "pml_bfo_hdr.h" - -BEGIN_C_DECLS - -bool mca_pml_bfo_is_duplicate_msg(mca_pml_bfo_comm_proc_t* proc, - mca_pml_bfo_match_hdr_t *hdr); -bool mca_pml_bfo_is_duplicate_fin(mca_pml_bfo_hdr_t* hdr, mca_btl_base_descriptor_t* rdma, - mca_btl_base_module_t* btl); - -mca_pml_bfo_recv_request_t* mca_pml_bfo_get_request(mca_pml_bfo_match_hdr_t *hdr); - -void mca_pml_bfo_send_request_restart(mca_pml_bfo_send_request_t* sendreq, - bool repost, mca_btl_base_tag_t tag); -void mca_pml_bfo_send_request_rndvrestartnotify(mca_pml_bfo_send_request_t* sendreq, - bool repost, mca_btl_base_tag_t tag, int status, - mca_btl_base_module_t* btl); - -void -mca_pml_bfo_rndvrestartnotify_completion(mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status); -void -mca_pml_bfo_check_recv_ctl_completion_status(mca_btl_base_module_t* btl, - struct mca_btl_base_descriptor_t* des, - int status); - -/* Reset a receive request to the beginning */ -void mca_pml_bfo_recv_request_reset(mca_pml_bfo_recv_request_t* recvreq); -/* Notify sender that receiver detected an error */ -void mca_pml_bfo_recv_request_recverrnotify(mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_tag_t tag, int status); -/* Ack the RNDVRESTARTNOTIFY message */ -void mca_pml_bfo_recv_request_rndvrestartack(mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_tag_t tag, int status, - mca_btl_base_module_t* btl); -/* Nack the RNDVRESTARTNOTIFY message */ -void mca_pml_bfo_recv_request_rndvrestartnack(mca_btl_base_descriptor_t* olddes, - ompi_proc_t* ompi_proc, bool repost); - -void mca_pml_bfo_recv_restart_completion(mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status); -void mca_pml_bfo_failover_error_handler(struct mca_btl_base_module_t* btl, - int32_t flags, ompi_proc_t *errproc, char *btlname); -void mca_pml_bfo_repost_match_fragment(struct mca_btl_base_descriptor_t* des); -void mca_pml_bfo_repost_fin(struct mca_btl_base_descriptor_t* des); - -void mca_pml_bfo_map_out_btl(struct mca_btl_base_module_t* btl, - ompi_proc_t *errproc, char *btlname); - -extern void mca_pml_bfo_map_out( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -int mca_pml_bfo_register_callbacks(void); - -void mca_pml_bfo_update_rndv_fields(mca_pml_bfo_hdr_t* hdr, - mca_pml_bfo_send_request_t*, char *type); - -void mca_pml_bfo_update_bml_btl(mca_bml_base_btl_t** bml_btl, mca_btl_base_module_t* btl, - struct mca_btl_base_descriptor_t* des); - -void mca_pml_bfo_find_recvreq_eager_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_recv_request_t* recvreq, - char* type); - -void mca_pml_bfo_find_sendreq_eager_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_send_request_t* sendreq, - char* type); - -void mca_pml_bfo_find_sendreq_rdma_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_send_request_t* sendreq, - char* type); - -void mca_pml_bfo_update_eager_bml_btl_recv_ctl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - struct mca_btl_base_descriptor_t* des); -void mca_pml_bfo_find_recvreq_rdma_bml_btl(mca_bml_base_btl_t** bml_btl, - mca_btl_base_module_t* btl, - mca_pml_bfo_recv_request_t* recvreq, - char* type); - -bool mca_pml_bfo_rndv_completion_status_error(struct mca_btl_base_descriptor_t* des, - mca_pml_bfo_send_request_t* sendreq); -void mca_pml_bfo_send_ctl_completion_status_error(struct mca_btl_base_descriptor_t* des); - - -void mca_pml_bfo_completion_sendreq_has_error(mca_pml_bfo_send_request_t* sendreq, - int status, - mca_btl_base_module_t* btl, - int type, - char *description); -/** - * Four new callbacks for the four new message types. - */ -extern void mca_pml_bfo_recv_frag_callback_rndvrestartnotify( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -extern void mca_pml_bfo_recv_frag_callback_rndvrestartack( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -extern void mca_pml_bfo_recv_frag_callback_rndvrestartnack( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -extern void mca_pml_bfo_recv_frag_callback_recverrnotify( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -/** - * A bunch of macros to help isolate failover code from regular ob1 code. - */ - -/* Drop any ACK fragments if request is in error state. Do not want - * to initiate any more activity. */ -#define MCA_PML_BFO_ERROR_CHECK_ON_ACK_CALLBACK(sendreq) \ - if( OPAL_UNLIKELY((sendreq)->req_error)) { \ - opal_output_verbose(20, mca_pml_bfo_output, \ - "ACK: received: dropping because request in error, " \ - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", \ - (uint16_t)(sendreq)->req_send.req_base.req_sequence, \ - (sendreq)->req_restartseq, \ - (void *)(sendreq), (sendreq)->req_recv.pval, \ - (sendreq)->req_send.req_base.req_peer); \ - return; \ - } - -/* Drop any FRAG fragments if request is in error state. Do not want - * to initiate any more activity. */ -#define MCA_PML_BFO_ERROR_CHECK_ON_FRAG_CALLBACK(recvreq) \ - if( OPAL_UNLIKELY((recvreq)->req_errstate)) { \ - opal_output_verbose(20, mca_pml_bfo_output, \ - "FRAG: received: dropping because request in error, " \ - "PML=%d, src_req=%p, dst_req=%p, peer=%d, offset=%d", \ - (uint16_t)(recvreq)->req_msgseq, \ - (recvreq)->remote_req_send.pval, \ - (void *)(recvreq), \ - (recvreq)->req_recv.req_base.req_ompi.req_status.MPI_SOURCE, \ - (int)hdr->hdr_frag.hdr_frag_offset); \ - return; \ - } - -/* Drop any PUT fragments if request is in error state. Do not want - * to initiate any more activity. */ -#define MCA_PML_BFO_ERROR_CHECK_ON_PUT_CALLBACK(sendreq) \ - if( OPAL_UNLIKELY((sendreq)->req_error)) { \ - opal_output_verbose(20, mca_pml_bfo_output, \ - "PUT: received: dropping because request in error, " \ - "PML=%d, src_req=%p, dst_req=%p, peer=%d", \ - (uint16_t)(sendreq)->req_send.req_base.req_sequence, \ - (void *)(sendreq), (sendreq)->req_recv.pval, \ - (sendreq)->req_send.req_base.req_peer); \ - return; \ - } - -/** - * Macros for pml_bfo_recvreq.c file. - */ - -/* This can happen if a FIN message arrives after the request was - * marked in error. So, just drop the message. Note that the status - * field is not being checked. That is because the status field is the - * value returned in the FIN hdr.hdr_fail field and may be used for - * other things. Note that we allow the various fields to be updated - * in case this actually completes the request and the sending side - * thinks it is done. */ -#define MCA_PML_BFO_ERROR_CHECK_ON_FIN_FOR_PUT(recvreq) \ - if( OPAL_UNLIKELY((recvreq)->req_errstate)) { \ - opal_output_verbose(20, mca_pml_bfo_output, \ - "FIN: received on broken request, skipping, " \ - "PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", \ - (recvreq)->req_msgseq, (recvreq)->req_restartseq, \ - (recvreq)->remote_req_send.pval, (void *)(recvreq), \ - (recvreq)->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); \ - /* Even though in error, it still might complete. */ \ - recv_request_pml_complete_check(recvreq); \ - return; \ - } - -#define MCA_PML_BFO_ERROR_CHECK_ON_RDMA_READ_COMPLETION(recvreq) \ - if ((recvreq)->req_errstate) { \ - opal_output_verbose(30, mca_pml_bfo_output, \ - "RDMA read: completion failed, error already seen, " \ - "PML=%d, RQS=%d, src_req=%lx, dst_req=%lx, peer=%d", \ - (recvreq)->req_msgseq, (recvreq)->req_restartseq, \ - (unsigned long)(recvreq)->remote_req_send.pval, \ - (unsigned long)(recvreq), \ - (recvreq)->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); \ - return; \ - } else { \ - opal_output_verbose(30, mca_pml_bfo_output, \ - "RDMA read: completion failed, sending RECVERRNOTIFY to " \ - "sender, PML=%d, RQS=%d, src_req=%lx, dst_req=%lx, peer=%d", \ - (recvreq)->req_msgseq, (recvreq)->req_restartseq, \ - (unsigned long)(recvreq)->remote_req_send.pval, \ - (unsigned long)(recvreq), \ - (recvreq)->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); \ - mca_pml_bfo_recv_request_recverrnotify(recvreq, MCA_PML_BFO_HDR_TYPE_RGET, status); \ - } - -#define MCA_PML_BFO_SECOND_ERROR_CHECK_ON_RDMA_READ_COMPLETION(recvreq, status, btl) \ - /* See if the request has received a RNDVRESTARTNOTIFY */ \ - if( OPAL_UNLIKELY(recvreq->req_errstate)) { \ - if (recvreq->req_errstate & RECVREQ_RNDVRESTART_RECVED) { \ - opal_output_verbose(30, mca_pml_bfo_output, \ - "RDMA read: completion: recvreq has error, outstanding events=%d " \ - "PML=%d, RQS=%d, src_req=%lx, dst_req=%lx, status=%d, peer=%d", \ - recvreq->req_events, recvreq->req_msgseq, recvreq->req_restartseq, \ - (unsigned long)recvreq->remote_req_send.pval, \ - (unsigned long)recvreq, status, \ - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); \ - if (0 == recvreq->req_events) { \ - mca_pml_bfo_recv_request_rndvrestartack(recvreq, MCA_PML_BFO_HDR_TYPE_RGET, \ - status, btl); \ - } \ - } \ - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); \ - return; \ - } - -/** - * Macros for pml_bfo_sendreq.c file. - */ - -/* This macro is called on the sending side after receiving - * a PUT message. There is a chance that this PUT message - * has shown up and is attempting to modify the state of - * the req_state, but the req_state is no longer being tracked - * because the RNDV message has turned into a RGET message - * because it got an error on the RNDV completion. - */ -#define MCA_PML_BFO_VERIFY_SENDREQ_REQ_STATE_VALUE(sendreq) \ - if (sendreq->req_state == -1) { \ - OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, 1); \ - } - -/* Now check the error state. This request can be in error if the - * RNDV message made it over, but the receiver got an error trying to - * send the ACK back and therefore sent a RECVERRNOTIFY message. In - * that case, we want to start the restart dance as the receiver has - * matched this message already. Only restart if there are no - * outstanding events on send request. */ -#define MCA_PML_BFO_RNDV_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl, type, description) \ - if( OPAL_UNLIKELY ((sendreq)->req_error)) { \ - mca_pml_bfo_completion_sendreq_has_error(sendreq, status, \ - btl, type, description); \ - return; \ - } - -/** - * This macro is called within the frag completion function in two - * places. It is called to see if any errors occur prior to the - * completion event on the frag. It is then called a second time - * after the scheduling routine is called as the scheduling routine - * may have detected that a BTL that was cached on the request had - * been removed and therefore marked the request in error. In that - * case, the scheduling of fragments can no longer proceed properly, - * and if there are no outstanding events, iniated the restart dance. - */ -#define MCA_PML_BFO_FRAG_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl, type, description) \ - if( OPAL_UNLIKELY((sendreq)->req_error)) { \ - mca_pml_bfo_completion_sendreq_has_error(sendreq, status, \ - btl, type, description); \ - return; \ - } - -/* This can happen if a FIN message arrives after the request was - * marked in error. So, just drop the message. Note that the status - * field is not checked here. That is because that is the value - * returned in the FIN hdr.hdr_fail field and may be used for other - * things. */ -#define MCA_PML_BFO_RGET_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, btl, des) \ - if( OPAL_UNLIKELY(sendreq->req_error)) { \ - opal_output_verbose(30, mca_pml_bfo_output, \ - "FIN: received on broken request, skipping, " \ - "PML=%d, src_req=%lx, dst_req=%lx, peer=%d", \ - (uint16_t)sendreq->req_send.req_base.req_sequence, \ - (unsigned long)sendreq, (unsigned long)sendreq->req_recv.pval, \ - sendreq->req_send.req_base.req_peer); \ - btl->btl_free(btl, des); \ - return; \ - } - - -/* Check if there has been an error on the send request when we get - * a completion event on the RDMA write. */ -#define MCA_PML_BFO_PUT_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl) \ - if ( OPAL_UNLIKELY(sendreq->req_error)) { \ - mca_pml_bfo_completion_sendreq_has_error(sendreq, status, btl, \ - MCA_PML_BFO_HDR_TYPE_PUT, "RDMA write"); \ - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); \ - return; \ - } - -#define MCA_PML_BFO_CHECK_FOR_RNDV_RESTART(hdr, sendreq, type) \ - if (0 < sendreq->req_restartseq) { \ - mca_pml_bfo_update_rndv_fields(hdr, sendreq, type); \ - } - -/* If a bml_btl gets mapped out, then we need to adjust it based - * on the btl from the callback function. These macros are called on - * every callback to make sure things are copacetic. - */ -#define MCA_PML_BFO_CHECK_EAGER_BML_BTL_ON_FIN_COMPLETION(bml_btl, btl, des) \ - if (bml_btl->btl != btl) { \ - ompi_proc_t *proc = (ompi_proc_t*) des->des_cbdata; \ - mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; \ - bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_eager, btl); \ - } -#define MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, type) \ - if (bml_btl->btl != btl) { \ - mca_pml_bfo_find_sendreq_eager_bml_btl(&bml_btl, btl, sendreq, type); \ - } -#define MCA_PML_BFO_CHECK_SENDREQ_RDMA_BML_BTL(bml_btl, btl, sendreq, type) \ - if (bml_btl->btl != btl) { \ - mca_pml_bfo_find_sendreq_rdma_bml_btl(&bml_btl, btl, sendreq, type); \ - } - -#define MCA_PML_BFO_CHECK_RECVREQ_EAGER_BML_BTL(bml_btl, btl, recvreq, type) \ - if (bml_btl->btl != btl) { \ - mca_pml_bfo_find_recvreq_eager_bml_btl(&bml_btl, btl, recvreq, type); \ - } - -#define MCA_PML_BFO_CHECK_RECVREQ_RDMA_BML_BTL(bml_btl, btl, recvreq, type) \ - if (bml_btl->btl != btl) { \ - mca_pml_bfo_find_recvreq_rdma_bml_btl(&bml_btl, btl, recvreq, type); \ - } - -#define MCA_PML_BFO_CHECK_RECVREQ_EAGER_BML_BTL_RECV_CTL(bml_btl, btl, des) \ - if (bml_btl->btl != btl) { \ - mca_pml_bfo_update_eager_bml_btl_recv_ctl(&bml_btl, btl, des); \ - } - -#define MCA_PML_BFO_CHECK_FOR_REMOVED_BML(sendreq, frag, btl) \ - if( OPAL_UNLIKELY(NULL == frag->rdma_bml) ) { \ - opal_output_verbose(30, mca_pml_bfo_output, \ - "PUT received: no matching BTL to RDMA write to, oustanding " \ - "events=%d, PML=%d, RQS=%d, src_req=%p, dst_req=%p, peer=%d", \ - sendreq->req_events, \ - (uint16_t)sendreq->req_send.req_base.req_sequence, \ - sendreq->req_restartseq, (void *)sendreq, \ - sendreq->req_recv.pval, sendreq->req_send.req_base.req_peer); \ - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); \ - sendreq->req_error++; \ - if (0 == sendreq->req_events) { \ - mca_pml_bfo_send_request_rndvrestartnotify(sendreq, false, \ - MCA_PML_BFO_HDR_TYPE_PUT, \ - OMPI_ERROR, btl); \ - } \ - return; \ - } - -/* This macro checks to see if the cached number of BTLs in the - * send request still matches the value from the endpoint. - * If it does not, this means that a BTL was removed from the - * available list. In this case, start the request over. - */ -#define MCA_PML_BFO_CHECK_FOR_REMOVED_BTL(sendreq, range) \ - if ((int)mca_bml_base_btl_array_get_size(&sendreq->req_endpoint->btl_send) \ - != range->range_btl_cnt) { \ - sendreq->req_error++; \ - return OMPI_ERROR; \ - } - - -END_C_DECLS - -#endif diff --git a/ompi/mca/pml/bfo/pml_bfo_hdr.h b/ompi/mca/pml/bfo/pml_bfo_hdr.h deleted file mode 100644 index 90a6f2326d..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_hdr.h +++ /dev/null @@ -1,539 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2009 IBM Corporation. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ -#ifndef MCA_PML_BFO_HEADER_H -#define MCA_PML_BFO_HEADER_H - -#include "ompi_config.h" -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif - -#include "opal/types.h" -#include "opal/util/arch.h" -#include "opal/mca/btl/btl.h" -#include "ompi/proc/proc.h" - -#define MCA_PML_BFO_HDR_TYPE_MATCH (MCA_BTL_TAG_PML + 1) -#define MCA_PML_BFO_HDR_TYPE_RNDV (MCA_BTL_TAG_PML + 2) -#define MCA_PML_BFO_HDR_TYPE_RGET (MCA_BTL_TAG_PML + 3) -#define MCA_PML_BFO_HDR_TYPE_ACK (MCA_BTL_TAG_PML + 4) -#define MCA_PML_BFO_HDR_TYPE_NACK (MCA_BTL_TAG_PML + 5) -#define MCA_PML_BFO_HDR_TYPE_FRAG (MCA_BTL_TAG_PML + 6) -#define MCA_PML_BFO_HDR_TYPE_GET (MCA_BTL_TAG_PML + 7) -#define MCA_PML_BFO_HDR_TYPE_PUT (MCA_BTL_TAG_PML + 8) -#define MCA_PML_BFO_HDR_TYPE_FIN (MCA_BTL_TAG_PML + 9) -#if PML_BFO -#define MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNOTIFY (MCA_BTL_TAG_PML + 10) -#define MCA_PML_BFO_HDR_TYPE_RNDVRESTARTACK (MCA_BTL_TAG_PML + 11) -#define MCA_PML_BFO_HDR_TYPE_RNDVRESTARTNACK (MCA_BTL_TAG_PML + 12) -#define MCA_PML_BFO_HDR_TYPE_RECVERRNOTIFY (MCA_BTL_TAG_PML + 13) -#endif /* PML_BFO */ - -#define MCA_PML_BFO_HDR_FLAGS_ACK 1 /* is an ack required */ -#define MCA_PML_BFO_HDR_FLAGS_NBO 2 /* is the hdr in network byte order */ -#define MCA_PML_BFO_HDR_FLAGS_PIN 4 /* is user buffer pinned */ -#define MCA_PML_BFO_HDR_FLAGS_CONTIG 8 /* is user buffer contiguous */ -#define MCA_PML_BFO_HDR_FLAGS_NORDMA 16 /* rest will be send by copy-in-out */ -#if PML_BFO -#define MCA_PML_BFO_HDR_FLAGS_RESTART 32 /* restart RNDV because of error */ -#endif /* PML_BFO */ - -/** - * Common hdr attributes - must be first element in each hdr type - */ -struct mca_pml_bfo_common_hdr_t { - uint8_t hdr_type; /**< type of envelope */ - uint8_t hdr_flags; /**< flags indicating how fragment should be processed */ -}; -typedef struct mca_pml_bfo_common_hdr_t mca_pml_bfo_common_hdr_t; - -#define MCA_PML_BFO_COMMON_HDR_NTOH(h) -#define MCA_PML_BFO_COMMON_HDR_HTON(h) - -/** - * Header definition for the first fragment, contains the - * attributes required to match the corresponding posted receive. - */ -struct mca_pml_bfo_match_hdr_t { - mca_pml_bfo_common_hdr_t hdr_common; /**< common attributes */ - uint16_t hdr_ctx; /**< communicator index */ - int32_t hdr_src; /**< source rank */ - int32_t hdr_tag; /**< user tag */ - uint16_t hdr_seq; /**< message sequence number */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[2]; /**< explicitly pad to 16 bytes. Compilers seem to already prefer to do this, but make it explicit just in case */ -#endif -}; -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT -#define OMPI_PML_BFO_MATCH_HDR_LEN 16 -#else -#define OMPI_PML_BFO_MATCH_HDR_LEN 14 -#endif - -typedef struct mca_pml_bfo_match_hdr_t mca_pml_bfo_match_hdr_t; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_MATCH_HDR_FILL(h) \ -do { \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ -} while(0) -#else -#define MCA_PML_BFO_MATCH_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -#define MCA_PML_BFO_MATCH_HDR_NTOH(h) \ -do { \ - MCA_PML_BFO_COMMON_HDR_NTOH((h).hdr_common); \ - (h).hdr_ctx = ntohs((h).hdr_ctx); \ - (h).hdr_src = ntohl((h).hdr_src); \ - (h).hdr_tag = ntohl((h).hdr_tag); \ - (h).hdr_seq = ntohs((h).hdr_seq); \ -} while (0) - -#define MCA_PML_BFO_MATCH_HDR_HTON(h) \ -do { \ - MCA_PML_BFO_COMMON_HDR_HTON((h).hdr_common); \ - MCA_PML_BFO_MATCH_HDR_FILL(h); \ - (h).hdr_ctx = htons((h).hdr_ctx); \ - (h).hdr_src = htonl((h).hdr_src); \ - (h).hdr_tag = htonl((h).hdr_tag); \ - (h).hdr_seq = htons((h).hdr_seq); \ -} while (0) - -/** - * Header definition for the first fragment when an acknowledgment - * is required. This could be the first fragment of a large message - * or a short message that requires an ack (synchronous). - */ -struct mca_pml_bfo_rendezvous_hdr_t { - mca_pml_bfo_match_hdr_t hdr_match; - uint64_t hdr_msg_length; /**< message length */ - opal_ptr_t hdr_src_req; /**< pointer to source request - returned in ack */ -#if PML_BFO - opal_ptr_t hdr_dst_req; /**< pointer to dst req */ - uint8_t hdr_restartseq; /**< restart sequence */ -#endif /* PML_BFO */ -}; -typedef struct mca_pml_bfo_rendezvous_hdr_t mca_pml_bfo_rendezvous_hdr_t; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_RNDV_HDR_FILL(h) \ - MCA_PML_BFO_MATCH_HDR_FILL((h).hdr_match) -#else -#define MCA_PML_BFO_RNDV_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -/* Note that hdr_src_req is not put in network byte order because it - is never processed by the receiver, other than being copied into - the ack header */ -#define MCA_PML_BFO_RNDV_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_MATCH_HDR_NTOH((h).hdr_match); \ - (h).hdr_msg_length = ntoh64((h).hdr_msg_length); \ - } while (0) - -#define MCA_PML_BFO_RNDV_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_MATCH_HDR_HTON((h).hdr_match); \ - MCA_PML_BFO_RNDV_HDR_FILL(h); \ - (h).hdr_msg_length = hton64((h).hdr_msg_length); \ - } while (0) - -/** - * Header definition for a combined rdma rendezvous/get - */ -struct mca_pml_bfo_rget_hdr_t { - mca_pml_bfo_rendezvous_hdr_t hdr_rndv; - uint32_t hdr_seg_cnt; /**< number of segments for rdma */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[4]; -#endif - opal_ptr_t hdr_des; /**< source descriptor */ -}; -typedef struct mca_pml_bfo_rget_hdr_t mca_pml_bfo_rget_hdr_t; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_RGET_HDR_FILL(h) \ -do { \ - MCA_PML_BFO_RNDV_HDR_FILL((h).hdr_rndv); \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ - (h).hdr_padding[2] = 0; \ - (h).hdr_padding[3] = 0; \ -} while(0) -#else -#define MCA_PML_BFO_RGET_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -#define MCA_PML_BFO_RGET_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_RNDV_HDR_NTOH((h).hdr_rndv); \ - (h).hdr_seg_cnt = ntohl((h).hdr_seg_cnt); \ - } while (0) - -#define MCA_PML_BFO_RGET_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_RNDV_HDR_HTON((h).hdr_rndv); \ - MCA_PML_BFO_RGET_HDR_FILL(h); \ - (h).hdr_seg_cnt = htonl((h).hdr_seg_cnt); \ - } while (0) - -/** - * Header for subsequent fragments. - */ -struct mca_pml_bfo_frag_hdr_t { - mca_pml_bfo_common_hdr_t hdr_common; /**< common attributes */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[6]; -#endif - uint64_t hdr_frag_offset; /**< offset into message */ - opal_ptr_t hdr_src_req; /**< pointer to source request */ - opal_ptr_t hdr_dst_req; /**< pointer to matched receive */ -}; -typedef struct mca_pml_bfo_frag_hdr_t mca_pml_bfo_frag_hdr_t; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_FRAG_HDR_FILL(h) \ -do { \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ - (h).hdr_padding[2] = 0; \ - (h).hdr_padding[3] = 0; \ - (h).hdr_padding[4] = 0; \ - (h).hdr_padding[5] = 0; \ -} while(0) -#else -#define MCA_PML_BFO_FRAG_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -#define MCA_PML_BFO_FRAG_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_NTOH((h).hdr_common); \ - (h).hdr_frag_offset = ntoh64((h).hdr_frag_offset); \ - } while (0) - -#define MCA_PML_BFO_FRAG_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_HTON((h).hdr_common); \ - MCA_PML_BFO_FRAG_HDR_FILL(h); \ - (h).hdr_frag_offset = hton64((h).hdr_frag_offset); \ - } while (0) - -/** - * Header used to acknowledgment outstanding fragment(s). - */ - -struct mca_pml_bfo_ack_hdr_t { - mca_pml_bfo_common_hdr_t hdr_common; /**< common attributes */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[6]; -#endif - opal_ptr_t hdr_src_req; /**< source request */ - opal_ptr_t hdr_dst_req; /**< matched receive request */ - uint64_t hdr_send_offset; /**< starting point of copy in/out */ -}; -typedef struct mca_pml_bfo_ack_hdr_t mca_pml_bfo_ack_hdr_t; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_ACK_HDR_FILL(h) \ -do { \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ - (h).hdr_padding[2] = 0; \ - (h).hdr_padding[3] = 0; \ - (h).hdr_padding[4] = 0; \ - (h).hdr_padding[5] = 0; \ -} while (0) -#else -#define MCA_PML_BFO_ACK_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -/* Note that the request headers are not put in NBO because the - src_req is already in receiver's byte order and the dst_req is not - used by the receiver for anything other than backpointers in return - headers */ -#define MCA_PML_BFO_ACK_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_NTOH((h).hdr_common); \ - (h).hdr_send_offset = ntoh64((h).hdr_send_offset); \ - } while (0) - -#define MCA_PML_BFO_ACK_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_HTON((h).hdr_common); \ - MCA_PML_BFO_ACK_HDR_FILL(h); \ - (h).hdr_send_offset = hton64((h).hdr_send_offset); \ - } while (0) - -/** - * Header used to initiate an RDMA operation. - */ - -struct mca_pml_bfo_rdma_hdr_t { - mca_pml_bfo_common_hdr_t hdr_common; /**< common attributes */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[2]; /** two to pad out the hdr to a 4 byte alignment. hdr_req will then be 8 byte aligned after 4 for hdr_seg_cnt */ -#endif - uint32_t hdr_seg_cnt; /**< number of segments for rdma */ - opal_ptr_t hdr_req; /**< destination request */ -#if PML_BFO - opal_ptr_t hdr_dst_req; /**< pointer to destination request */ -#endif /* PML_BFO */ - opal_ptr_t hdr_des; /**< source descriptor */ - uint64_t hdr_rdma_offset; /**< current offset into user buffer */ - mca_btl_base_segment_t hdr_segs[1]; /**< list of segments for rdma */ -}; -typedef struct mca_pml_bfo_rdma_hdr_t mca_pml_bfo_rdma_hdr_t; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_RDMA_HDR_FILL(h) \ -do { \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ -} while(0) -#else -#define MCA_PML_BFO_RDMA_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -#define MCA_PML_BFO_RDMA_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_NTOH((h).hdr_common); \ - (h).hdr_seg_cnt = ntohl((h).hdr_seg_cnt); \ - (h).hdr_rdma_offset = ntoh64((h).hdr_rdma_offset); \ - } while (0) - -#define MCA_PML_BFO_RDMA_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_HTON((h).hdr_common); \ - MCA_PML_BFO_RDMA_HDR_FILL(h); \ - (h).hdr_seg_cnt = htonl((h).hdr_seg_cnt); \ - (h).hdr_rdma_offset = hton64((h).hdr_rdma_offset); \ - } while (0) - -/** - * Header used to complete an RDMA operation. - */ - -struct mca_pml_bfo_fin_hdr_t { - mca_pml_bfo_common_hdr_t hdr_common; /**< common attributes */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[2]; -#endif -#if PML_BFO - /* Match info is needed to check for duplicate FIN messages. */ - mca_pml_bfo_match_hdr_t hdr_match; -#endif /* PML_BFO */ - uint32_t hdr_fail; /**< RDMA operation failed */ - opal_ptr_t hdr_des; /**< completed descriptor */ -}; -typedef struct mca_pml_bfo_fin_hdr_t mca_pml_bfo_fin_hdr_t; - -#if PML_BFO -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_FIN_HDR_FILL(h) \ -do { \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ - MCA_PML_BFO_MATCH_HDR_FILL((h).hdr_match); \ -} while (0) -#else -#define MCA_PML_BFO_FIN_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -#define MCA_PML_BFO_FIN_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_NTOH((h).hdr_common); \ - MCA_PML_BFO_MATCH_HDR_NTOH((h).hdr_match); \ - } while (0) - -#define MCA_PML_BFO_FIN_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_HTON((h).hdr_common); \ - MCA_PML_BFO_MATCH_HDR_HTON((h).hdr_match); \ - MCA_PML_BFO_FIN_HDR_FILL(h); \ - } while (0) -#else /* PML_BFO */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG -#define MCA_PML_BFO_FIN_HDR_FILL(h) \ -do { \ - (h).hdr_padding[0] = 0; \ - (h).hdr_padding[1] = 0; \ -} while (0) -#else -#define MCA_PML_BFO_FIN_HDR_FILL(h) -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT && OPAL_ENABLE_DEBUG */ - -#define MCA_PML_BFO_FIN_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_NTOH((h).hdr_common); \ - } while (0) - -#define MCA_PML_BFO_FIN_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_COMMON_HDR_HTON((h).hdr_common); \ - MCA_PML_BFO_FIN_HDR_FILL(h); \ - } while (0) -#endif /* PML_BFO */ - -#if PML_BFO -/** - * Header used to restart a rendezvous request. - */ -struct mca_pml_bfo_restart_hdr_t { - mca_pml_bfo_match_hdr_t hdr_match; /**< needed to avoid duplicate messages */ - uint8_t hdr_restartseq; /**< restart sequence */ -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - uint8_t hdr_padding[3]; -#endif - opal_ptr_t hdr_src_req; /**< source request */ - opal_ptr_t hdr_dst_req; /**< matched receive request */ - int32_t hdr_dst_rank; /**< needed to send NACK */ - uint32_t hdr_jobid; /**< needed to send NACK */ - uint32_t hdr_vpid; /**< needed to send NACK */ -}; -typedef struct mca_pml_bfo_restart_hdr_t mca_pml_bfo_restart_hdr_t; - -/* Only need to put parts of the restart header in NBO. No need - to do hdr_src_req and hdr_dst_req as they are only used on the - by the process that originated them. */ -#define MCA_PML_BFO_RESTART_HDR_NTOH(h) \ - do { \ - MCA_PML_BFO_MATCH_HDR_NTOH((h).hdr_match); \ - (h).hdr_dst_rank = ntohl((h).hdr_dst_rank); \ - (h).hdr_jobid = ntohl((h).hdr_jobid); \ - (h).hdr_vpid = ntohl((h).hdr_vpid); \ - } while (0) - -#define MCA_PML_BFO_RESTART_HDR_HTON(h) \ - do { \ - MCA_PML_BFO_MATCH_HDR_HTON((h).hdr_match); \ - (h).hdr_dst_rank = htonl((h).hdr_dst_rank); \ - (h).hdr_jobid = htonl((h).hdr_jobid); \ - (h).hdr_vpid = htonl((h).hdr_vpid); \ - } while (0) - -#endif /* PML_BFO */ -/** - * Union of defined hdr types. - */ -union mca_pml_bfo_hdr_t { - mca_pml_bfo_common_hdr_t hdr_common; - mca_pml_bfo_match_hdr_t hdr_match; - mca_pml_bfo_rendezvous_hdr_t hdr_rndv; - mca_pml_bfo_rget_hdr_t hdr_rget; - mca_pml_bfo_frag_hdr_t hdr_frag; - mca_pml_bfo_ack_hdr_t hdr_ack; - mca_pml_bfo_rdma_hdr_t hdr_rdma; - mca_pml_bfo_fin_hdr_t hdr_fin; -#if PML_BFO - mca_pml_bfo_restart_hdr_t hdr_restart; -#endif /* PML_BFO */ -}; -typedef union mca_pml_bfo_hdr_t mca_pml_bfo_hdr_t; - -#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT -static inline __opal_attribute_always_inline__ void -bfo_hdr_ntoh(mca_pml_bfo_hdr_t *hdr, const uint8_t hdr_type) -{ - if(!(hdr->hdr_common.hdr_flags & MCA_PML_BFO_HDR_FLAGS_NBO)) - return; - - switch(hdr_type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - MCA_PML_BFO_MATCH_HDR_NTOH(hdr->hdr_match); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - MCA_PML_BFO_RNDV_HDR_NTOH(hdr->hdr_rndv); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - MCA_PML_BFO_RGET_HDR_NTOH(hdr->hdr_rget); - break; - case MCA_PML_BFO_HDR_TYPE_ACK: - MCA_PML_BFO_ACK_HDR_NTOH(hdr->hdr_ack); - break; - case MCA_PML_BFO_HDR_TYPE_FRAG: - MCA_PML_BFO_FRAG_HDR_NTOH(hdr->hdr_frag); - break; - case MCA_PML_BFO_HDR_TYPE_PUT: - MCA_PML_BFO_RDMA_HDR_NTOH(hdr->hdr_rdma); - break; - case MCA_PML_BFO_HDR_TYPE_FIN: - MCA_PML_BFO_FIN_HDR_NTOH(hdr->hdr_fin); - break; - default: - assert(0); - break; - } -} -#else -#define bfo_hdr_ntoh(h, t) do{}while(0) -#endif - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT -#define bfo_hdr_hton(h, t, p) \ - bfo_hdr_hton_intr((mca_pml_bfo_hdr_t*)h, t, p) -static inline __opal_attribute_always_inline__ void -bfo_hdr_hton_intr(mca_pml_bfo_hdr_t *hdr, const uint8_t hdr_type, - const ompi_proc_t *proc) -{ -#ifdef WORDS_BIGENDIAN - hdr->hdr_common.hdr_flags |= MCA_PML_BFO_HDR_FLAGS_NBO; -#else - - if(!(proc->super.proc_arch & OPAL_ARCH_ISBIGENDIAN)) - return; - - hdr->hdr_common.hdr_flags |= MCA_PML_BFO_HDR_FLAGS_NBO; - switch(hdr_type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - MCA_PML_BFO_MATCH_HDR_HTON(hdr->hdr_match); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - MCA_PML_BFO_RNDV_HDR_HTON(hdr->hdr_rndv); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - MCA_PML_BFO_RGET_HDR_HTON(hdr->hdr_rget); - break; - case MCA_PML_BFO_HDR_TYPE_ACK: - MCA_PML_BFO_ACK_HDR_HTON(hdr->hdr_ack); - break; - case MCA_PML_BFO_HDR_TYPE_FRAG: - MCA_PML_BFO_FRAG_HDR_HTON(hdr->hdr_frag); - break; - case MCA_PML_BFO_HDR_TYPE_PUT: - MCA_PML_BFO_RDMA_HDR_HTON(hdr->hdr_rdma); - break; - case MCA_PML_BFO_HDR_TYPE_FIN: - MCA_PML_BFO_FIN_HDR_HTON(hdr->hdr_fin); - break; - default: - assert(0); - break; - } -#endif -} -#else -#define bfo_hdr_hton(h, t, p) do{}while(0) -#endif -#endif diff --git a/ompi/mca/pml/bfo/pml_bfo_iprobe.c b/ompi/mca/pml/bfo/pml_bfo_iprobe.c deleted file mode 100644 index c3d432367a..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_iprobe.c +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "ompi/request/request.h" -#include "ompi/message/message.h" -#include "pml_bfo_recvreq.h" - - -int mca_pml_bfo_iprobe(int src, - int tag, - struct ompi_communicator_t *comm, - int *matched, ompi_status_public_t * status) -{ - int rc = OMPI_SUCCESS; - mca_pml_bfo_recv_request_t recvreq; - - OBJ_CONSTRUCT( &recvreq, mca_pml_bfo_recv_request_t ); - recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML; - recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_IPROBE; - - MCA_PML_BFO_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char.dt, src, tag, comm, false); - MCA_PML_BFO_RECV_REQUEST_START(&recvreq); - - if( recvreq.req_recv.req_base.req_ompi.req_complete == true ) { - if( NULL != status ) { - *status = recvreq.req_recv.req_base.req_ompi.req_status; - } - rc = recvreq.req_recv.req_base.req_ompi.req_status.MPI_ERROR; - *matched = 1; - } else { - *matched = 0; - opal_progress(); - } - MCA_PML_BASE_RECV_REQUEST_FINI( &recvreq.req_recv ); - return rc; -} - - -int mca_pml_bfo_probe(int src, - int tag, - struct ompi_communicator_t *comm, - ompi_status_public_t * status) -{ - int rc = OMPI_SUCCESS; - mca_pml_bfo_recv_request_t recvreq; - - OBJ_CONSTRUCT( &recvreq, mca_pml_bfo_recv_request_t ); - recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML; - recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_PROBE; - - MCA_PML_BFO_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char.dt, src, tag, comm, false); - MCA_PML_BFO_RECV_REQUEST_START(&recvreq); - - ompi_request_wait_completion(&recvreq.req_recv.req_base.req_ompi); - rc = recvreq.req_recv.req_base.req_ompi.req_status.MPI_ERROR; - if (NULL != status) { - *status = recvreq.req_recv.req_base.req_ompi.req_status; - } - - MCA_PML_BASE_RECV_REQUEST_FINI( &recvreq.req_recv ); - return rc; -} - - -int -mca_pml_bfo_improbe(int src, - int tag, - struct ompi_communicator_t *comm, - int *matched, - struct ompi_message_t **message, - ompi_status_public_t * status) -{ - int rc = OMPI_SUCCESS; - mca_pml_bfo_recv_request_t *recvreq; - - *message = ompi_message_alloc(); - if (NULL == *message) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - - MCA_PML_BFO_RECV_REQUEST_ALLOC(recvreq); - if (NULL == recvreq) - return OMPI_ERR_OUT_OF_RESOURCE; - recvreq->req_recv.req_base.req_type = MCA_PML_REQUEST_IMPROBE; - - /* initialize the request enough to probe and get the status */ - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, NULL, 0, &ompi_mpi_char.dt, - src, tag, comm, false); - MCA_PML_BFO_RECV_REQUEST_START(recvreq); - - if( recvreq->req_recv.req_base.req_ompi.req_complete == true ) { - if( NULL != status ) { - *status = recvreq->req_recv.req_base.req_ompi.req_status; - } - *matched = 1; - - (*message)->comm = comm; - (*message)->req_ptr = recvreq; - (*message)->peer = recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE; - (*message)->count = recvreq->req_recv.req_base.req_ompi.req_status._ucount; - - rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR; - } else { - *matched = 0; - - /* we only free if we didn't match, because we're going to - translate the request into a receive request later on if it - was matched */ - MCA_PML_BFO_RECV_REQUEST_RETURN( recvreq ); - ompi_message_return(*message); - *message = MPI_MESSAGE_NULL; - - opal_progress(); - } - - return rc; -} - - -int -mca_pml_bfo_mprobe(int src, - int tag, - struct ompi_communicator_t *comm, - struct ompi_message_t **message, - ompi_status_public_t * status) -{ - int rc = OMPI_SUCCESS; - mca_pml_bfo_recv_request_t *recvreq; - - *message = ompi_message_alloc(); - if (NULL == *message) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - - MCA_PML_BFO_RECV_REQUEST_ALLOC(recvreq); - if (NULL == recvreq) - return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - recvreq->req_recv.req_base.req_type = MCA_PML_REQUEST_MPROBE; - - /* initialize the request enough to probe and get the status */ - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, NULL, 0, &ompi_mpi_char.dt, - src, tag, comm, false); - MCA_PML_BFO_RECV_REQUEST_START(recvreq); - - ompi_request_wait_completion(&recvreq->req_recv.req_base.req_ompi); - rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR; - - if( NULL != status ) { - *status = recvreq->req_recv.req_base.req_ompi.req_status; - } - - (*message)->comm = comm; - (*message)->req_ptr = recvreq; - (*message)->peer = recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE; - (*message)->count = recvreq->req_recv.req_base.req_ompi.req_status._ucount; - - return rc; -} diff --git a/ompi/mca/pml/bfo/pml_bfo_irecv.c b/ompi/mca/pml/bfo/pml_bfo_irecv.c deleted file mode 100644 index b4b2cb352b..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_irecv.c +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "ompi/request/request.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_recvfrag.h" -#include "ompi/peruse/peruse-internal.h" -#include "ompi/message/message.h" - -int mca_pml_bfo_irecv_init(void *addr, - size_t count, - ompi_datatype_t * datatype, - int src, - int tag, - struct ompi_communicator_t *comm, - struct ompi_request_t **request) -{ - mca_pml_bfo_recv_request_t *recvreq; - MCA_PML_BFO_RECV_REQUEST_ALLOC(recvreq); - if (NULL == recvreq) - return OMPI_ERR_OUT_OF_RESOURCE; - - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, - addr, - count, datatype, src, tag, comm, true); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &((recvreq)->req_recv.req_base), - PERUSE_RECV); - - *request = (ompi_request_t *) recvreq; - return OMPI_SUCCESS; -} - -int mca_pml_bfo_irecv(void *addr, - size_t count, - ompi_datatype_t * datatype, - int src, - int tag, - struct ompi_communicator_t *comm, - struct ompi_request_t **request) -{ - mca_pml_bfo_recv_request_t *recvreq; - MCA_PML_BFO_RECV_REQUEST_ALLOC(recvreq); - if (NULL == recvreq) - return OMPI_ERR_OUT_OF_RESOURCE; - - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, - addr, - count, datatype, src, tag, comm, false); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &((recvreq)->req_recv.req_base), - PERUSE_RECV); - - MCA_PML_BFO_RECV_REQUEST_START(recvreq); - *request = (ompi_request_t *) recvreq; - return OMPI_SUCCESS; -} - - -int mca_pml_bfo_recv(void *addr, - size_t count, - ompi_datatype_t * datatype, - int src, - int tag, - struct ompi_communicator_t *comm, - ompi_status_public_t * status) -{ - int rc; - mca_pml_bfo_recv_request_t *recvreq; - MCA_PML_BFO_RECV_REQUEST_ALLOC(recvreq); - if (NULL == recvreq) - return OMPI_ERR_OUT_OF_RESOURCE; - - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, - addr, - count, datatype, src, tag, comm, false); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &((recvreq)->req_recv.req_base), - PERUSE_RECV); - - MCA_PML_BFO_RECV_REQUEST_START(recvreq); - ompi_request_wait_completion(&recvreq->req_recv.req_base.req_ompi); - - if (NULL != status) { /* return status */ - *status = recvreq->req_recv.req_base.req_ompi.req_status; - } - rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR; - ompi_request_free( (ompi_request_t**)&recvreq ); - return rc; -} - - -int -mca_pml_bfo_imrecv( void *buf, - size_t count, - ompi_datatype_t *datatype, - struct ompi_message_t **message, - struct ompi_request_t **request ) -{ - mca_pml_bfo_recv_frag_t* frag; - mca_pml_bfo_recv_request_t *recvreq; - mca_pml_bfo_hdr_t *hdr; - int src, tag; - ompi_communicator_t *comm; - mca_pml_bfo_comm_proc_t* proc; - mca_pml_bfo_comm_t* bfo_comm; - uint64_t seq; - - /* get the request from the message and the frag from the request - before we overwrite everything */ - recvreq = (mca_pml_bfo_recv_request_t*) (*message)->req_ptr; - frag = (mca_pml_bfo_recv_frag_t*) recvreq->req_recv.req_base.req_addr; - src = recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE; - tag = recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG; - comm = (*message)->comm; - bfo_comm = recvreq->req_recv.req_base.req_comm->c_pml_comm; - seq = recvreq->req_recv.req_base.req_sequence; - - /* make the request a recv request again */ - /* The old request kept pointers to comm and the char datatype. - We're about to release those, but need to make sure comm - doesn't go out of scope (we don't care about the char datatype - anymore). So retain comm, then release the frag, then reinit - the frag (which will retain comm), then release comm (but the - frag still has it's ref, so it'll stay in scope). Make - sense? */ - OBJ_RETAIN(comm); - MCA_PML_BASE_RECV_REQUEST_FINI(&recvreq->req_recv); - recvreq->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV; - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, - buf, - count, datatype, - src, tag, comm, false); - OBJ_RELEASE(comm); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &((recvreq)->req_recv.req_base), - PERUSE_RECV); - - /* init/re-init the request */ - recvreq->req_lock = 0; - recvreq->req_pipeline_depth = 0; - recvreq->req_bytes_received = 0; - /* What about req_rdma_cnt ? */ - recvreq->req_rdma_idx = 0; - recvreq->req_pending = false; - recvreq->req_ack_sent = false; - - MCA_PML_BASE_RECV_START(&recvreq->req_recv.req_base); - - /* Note - sequence number already assigned */ - recvreq->req_recv.req_base.req_sequence = seq; - - proc = &bfo_comm->procs[recvreq->req_recv.req_base.req_peer]; - recvreq->req_recv.req_base.req_proc = proc->ompi_proc; - prepare_recv_req_converter(recvreq); - - /* we can't go through the match, since we already have the match. - Cheat and do what REQUEST_START does, but without the frag - search */ - hdr = (mca_pml_bfo_hdr_t*)frag->segments->seg_addr.pval; - switch(hdr->hdr_common.hdr_type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - mca_pml_bfo_recv_request_progress_match(recvreq, frag->btl, frag->segments, - frag->num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - mca_pml_bfo_recv_request_progress_rndv(recvreq, frag->btl, frag->segments, - frag->num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - mca_pml_bfo_recv_request_progress_rget(recvreq, frag->btl, frag->segments, - frag->num_segments); - break; - default: - assert(0); - } - MCA_PML_BFO_RECV_FRAG_RETURN(frag); - - ompi_message_return(*message); - *message = MPI_MESSAGE_NULL; - *request = (ompi_request_t *) recvreq; - - return OMPI_SUCCESS; -} - - -int -mca_pml_bfo_mrecv( void *buf, - size_t count, - ompi_datatype_t *datatype, - struct ompi_message_t **message, - ompi_status_public_t* status ) -{ - mca_pml_bfo_recv_frag_t* frag; - mca_pml_bfo_recv_request_t *recvreq; - mca_pml_bfo_hdr_t *hdr; - int src, tag, rc; - ompi_communicator_t *comm; - mca_pml_bfo_comm_proc_t* proc; - mca_pml_bfo_comm_t* bfo_comm; - uint64_t seq; - - /* get the request from the message and the frag from the request - before we overwrite everything */ - comm = (*message)->comm; - recvreq = (mca_pml_bfo_recv_request_t*) (*message)->req_ptr; - frag = (mca_pml_bfo_recv_frag_t*) recvreq->req_recv.req_base.req_addr; - src = recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE; - tag = recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG; - seq = recvreq->req_recv.req_base.req_sequence; - bfo_comm = recvreq->req_recv.req_base.req_comm->c_pml_comm; - - /* make the request a recv request again */ - /* The old request kept pointers to comm and the char datatype. - We're about to release those, but need to make sure comm - doesn't go out of scope (we don't care about the char datatype - anymore). So retain comm, then release the frag, then reinit - the frag (which will retain comm), then release comm (but the - frag still has it's ref, so it'll stay in scope). Make - sense? */ - OBJ_RETAIN(comm); - MCA_PML_BASE_RECV_REQUEST_FINI(&recvreq->req_recv); - recvreq->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV; - MCA_PML_BFO_RECV_REQUEST_INIT(recvreq, - buf, - count, datatype, - src, tag, comm, false); - OBJ_RELEASE(comm); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &((recvreq)->req_recv.req_base), - PERUSE_RECV); - - /* init/re-init the request */ - recvreq->req_lock = 0; - recvreq->req_pipeline_depth = 0; - recvreq->req_bytes_received = 0; - recvreq->req_rdma_cnt = 0; - recvreq->req_rdma_idx = 0; - recvreq->req_pending = false; - - MCA_PML_BASE_RECV_START(&recvreq->req_recv.req_base); - - /* Note - sequence number already assigned */ - recvreq->req_recv.req_base.req_sequence = seq; - - proc = &bfo_comm->procs[recvreq->req_recv.req_base.req_peer]; - recvreq->req_recv.req_base.req_proc = proc->ompi_proc; - prepare_recv_req_converter(recvreq); - - /* we can't go through the match, since we already have the match. - Cheat and do what REQUEST_START does, but without the frag - search */ - hdr = (mca_pml_bfo_hdr_t*)frag->segments->seg_addr.pval; - switch(hdr->hdr_common.hdr_type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - mca_pml_bfo_recv_request_progress_match(recvreq, frag->btl, frag->segments, - frag->num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - mca_pml_bfo_recv_request_progress_rndv(recvreq, frag->btl, frag->segments, - frag->num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - mca_pml_bfo_recv_request_progress_rget(recvreq, frag->btl, frag->segments, - frag->num_segments); - break; - default: - assert(0); - } - - ompi_message_return(*message); - *message = MPI_MESSAGE_NULL; - ompi_request_wait_completion(&(recvreq->req_recv.req_base.req_ompi)); - - MCA_PML_BFO_RECV_FRAG_RETURN(frag); - - if (NULL != status) { /* return status */ - *status = recvreq->req_recv.req_base.req_ompi.req_status; - } - rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR; - ompi_request_free( (ompi_request_t**)&recvreq ); - return rc; -} - diff --git a/ompi/mca/pml/bfo/pml_bfo_isend.c b/ompi/mca/pml/bfo/pml_bfo_isend.c deleted file mode 100644 index 599d1afc0a..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_isend.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "pml_bfo.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_recvreq.h" -#include "ompi/peruse/peruse-internal.h" - -int mca_pml_bfo_isend_init(void *buf, - size_t count, - ompi_datatype_t * datatype, - int dst, - int tag, - mca_pml_base_send_mode_t sendmode, - ompi_communicator_t * comm, - ompi_request_t ** request) -{ - mca_pml_bfo_send_request_t *sendreq = NULL; - MCA_PML_BFO_SEND_REQUEST_ALLOC(comm, dst, sendreq); - if (NULL == sendreq) - return OMPI_ERR_OUT_OF_RESOURCE; - - MCA_PML_BFO_SEND_REQUEST_INIT(sendreq, - buf, - count, - datatype, - dst, tag, - comm, sendmode, true); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &(sendreq)->req_send.req_base, - PERUSE_SEND); - - *request = (ompi_request_t *) sendreq; - return OMPI_SUCCESS; -} - - -int mca_pml_bfo_isend(void *buf, - size_t count, - ompi_datatype_t * datatype, - int dst, - int tag, - mca_pml_base_send_mode_t sendmode, - ompi_communicator_t * comm, - ompi_request_t ** request) -{ - int rc; - mca_pml_bfo_send_request_t *sendreq = NULL; - - MCA_PML_BFO_SEND_REQUEST_ALLOC(comm, dst, sendreq); - if (NULL == sendreq) - return OMPI_ERR_OUT_OF_RESOURCE; - - MCA_PML_BFO_SEND_REQUEST_INIT(sendreq, - buf, - count, - datatype, - dst, tag, - comm, sendmode, false); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &(sendreq)->req_send.req_base, - PERUSE_SEND); - - MCA_PML_BFO_SEND_REQUEST_START(sendreq, rc); - *request = (ompi_request_t *) sendreq; - return rc; -} - - -int mca_pml_bfo_send(void *buf, - size_t count, - ompi_datatype_t * datatype, - int dst, - int tag, - mca_pml_base_send_mode_t sendmode, - ompi_communicator_t * comm) -{ - int rc; - mca_pml_bfo_send_request_t *sendreq; - - MCA_PML_BFO_SEND_REQUEST_ALLOC(comm, dst, sendreq); - if (NULL == sendreq) - return OMPI_ERR_OUT_OF_RESOURCE; - - MCA_PML_BFO_SEND_REQUEST_INIT(sendreq, - buf, - count, - datatype, - dst, tag, - comm, sendmode, false); - - PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE, - &(sendreq)->req_send.req_base, - PERUSE_SEND); - - MCA_PML_BFO_SEND_REQUEST_START(sendreq, rc); - if (rc != OMPI_SUCCESS) { - MCA_PML_BFO_SEND_REQUEST_RETURN( sendreq ); - return rc; - } - - ompi_request_wait_completion(&sendreq->req_send.req_base.req_ompi); - - rc = sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR; - ompi_request_free( (ompi_request_t**)&sendreq ); - return rc; -} diff --git a/ompi/mca/pml/bfo/pml_bfo_progress.c b/ompi/mca/pml/bfo/pml_bfo_progress.c deleted file mode 100644 index 1ab217357b..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_progress.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "pml_bfo.h" -#include "pml_bfo_sendreq.h" -#include "ompi/mca/bml/base/base.h" - -int mca_pml_bfo_progress(void) -{ - int i, queue_length = opal_list_get_size(&mca_pml_bfo.send_pending); - int j, completed_requests = 0; - bool send_succedded; - - if( OPAL_LIKELY(0 == queue_length) ) - return 0; - - for( i = 0; i < queue_length; i++ ) { - mca_pml_bfo_send_pending_t pending_type = MCA_PML_BFO_SEND_PENDING_NONE; - mca_pml_bfo_send_request_t* sendreq; - mca_bml_base_endpoint_t* endpoint; - - sendreq = get_request_from_send_pending(&pending_type); - if(OPAL_UNLIKELY(NULL == sendreq)) - break; - - switch(pending_type) { - case MCA_PML_BFO_SEND_PENDING_NONE: - assert(0); - return 0; - case MCA_PML_BFO_SEND_PENDING_SCHEDULE: - if( mca_pml_bfo_send_request_schedule_exclusive(sendreq) == - OMPI_ERR_OUT_OF_RESOURCE ) { - return 0; - } - completed_requests++; - break; - case MCA_PML_BFO_SEND_PENDING_START: - endpoint = sendreq->req_endpoint; - send_succedded = false; - for(j = 0; j < (int)mca_bml_base_btl_array_get_size(&endpoint->btl_eager); j++) { - mca_bml_base_btl_t* bml_btl; - int rc; - - /* select a btl */ - bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager); - rc = mca_pml_bfo_send_request_start_btl(sendreq, bml_btl); - if( OPAL_LIKELY(OMPI_SUCCESS == rc) ) { - send_succedded = true; - completed_requests++; - break; - } - } - if( false == send_succedded ) { - add_request_to_send_pending(sendreq, MCA_PML_BFO_SEND_PENDING_START, true); - } - } - } - return completed_requests; -} - diff --git a/ompi/mca/pml/bfo/pml_bfo_rdma.c b/ompi/mca/pml/bfo/pml_bfo_rdma.c deleted file mode 100644 index 0bd9984907..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_rdma.c +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/ - -#include "ompi_config.h" -#include "ompi/constants.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/bml/bml.h" -#include "opal/mca/mpool/mpool.h" -#include "pml_bfo.h" -#include "pml_bfo_rdma.h" - -/* Use this registration if no registration needed for a BTL instead of NULL. - * This will help other code to distinguish case when memory is not registered - * from case when registration is not needed */ -static mca_mpool_base_registration_t pml_bfo_dummy_reg; - -/* - * Check to see if memory is registered or can be registered. Build a - * set of registrations on the request. - */ - -size_t mca_pml_bfo_rdma_btls( - mca_bml_base_endpoint_t* bml_endpoint, - unsigned char* base, - size_t size, - mca_pml_bfo_com_btl_t* rdma_btls) -{ - int num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_rdma); - double weight_total = 0; - int num_btls_used = 0, n; - - /* shortcut when there are no rdma capable btls */ - if(num_btls == 0) { - return 0; - } - - /* check to see if memory is registered */ - for(n = 0; n < num_btls && num_btls_used < mca_pml_bfo.max_rdma_per_request; - n++) { - mca_bml_base_btl_t* bml_btl = - mca_bml_base_btl_array_get_index(&bml_endpoint->btl_rdma, - (bml_endpoint->btl_rdma_index + n) % num_btls); - mca_mpool_base_registration_t* reg = &pml_bfo_dummy_reg; - mca_mpool_base_module_t *btl_mpool = bml_btl->btl->btl_mpool; - - if( NULL != btl_mpool ) { - if(!mca_pml_bfo.leave_pinned) { - /* look through existing registrations */ - btl_mpool->mpool_find(btl_mpool, base, size, ®); - } else { - /* register the memory */ - btl_mpool->mpool_register(btl_mpool, base, size, 0, ®); - } - - if(NULL == reg) - continue; - } - - rdma_btls[num_btls_used].bml_btl = bml_btl; - rdma_btls[num_btls_used].btl_reg = reg; - weight_total += bml_btl->btl_weight; - num_btls_used++; - } - - /* if we don't use leave_pinned and all BTLs that already have this memory - * registered amount to less then half of available bandwidth - fall back to - * pipeline protocol */ - if(0 == num_btls_used || (!mca_pml_bfo.leave_pinned && weight_total < 0.5)) - return 0; - - mca_pml_bfo_calc_weighted_length(rdma_btls, num_btls_used, size, - weight_total); - - bml_endpoint->btl_rdma_index = (bml_endpoint->btl_rdma_index + 1) % num_btls; - return num_btls_used; -} - -size_t mca_pml_bfo_rdma_pipeline_btls( mca_bml_base_endpoint_t* bml_endpoint, - size_t size, - mca_pml_bfo_com_btl_t* rdma_btls ) -{ - int i, num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_rdma); - double weight_total = 0; - - for(i = 0; i < num_btls && i < mca_pml_bfo.max_rdma_per_request; i++) { - rdma_btls[i].bml_btl = - mca_bml_base_btl_array_get_next(&bml_endpoint->btl_rdma); - if(NULL != rdma_btls[i].bml_btl->btl->btl_mpool) - rdma_btls[i].btl_reg = NULL; - else - rdma_btls[i].btl_reg = &pml_bfo_dummy_reg; - - weight_total += rdma_btls[i].bml_btl->btl_weight; - } - - mca_pml_bfo_calc_weighted_length(rdma_btls, i, size, weight_total); - - return i; -} diff --git a/ompi/mca/pml/bfo/pml_bfo_rdma.h b/ompi/mca/pml/bfo/pml_bfo_rdma.h deleted file mode 100644 index abeb65a0f8..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_rdma.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ - -#ifndef MCA_PML_BFO_RDMA_H -#define MCA_PML_BFO_RDMA_H - -struct mca_bml_base_endpoint_t; - -/* - * Of the set of available btls that support RDMA, - * find those that already have registrations - or - * register if required (for leave_pinned option) - */ -size_t mca_pml_bfo_rdma_btls(struct mca_bml_base_endpoint_t* endpoint, - unsigned char* base, size_t size, struct mca_pml_bfo_com_btl_t* btls); - -/* Choose RDMA BTLs to use for sending of a request by pipeline protocol. - * Calculate number of bytes to send through each BTL according to available - * bandwidth */ -size_t mca_pml_bfo_rdma_pipeline_btls(struct mca_bml_base_endpoint_t* endpoint, - size_t size, mca_pml_bfo_com_btl_t* rdma_btls); -#endif - diff --git a/ompi/mca/pml/bfo/pml_bfo_rdmafrag.c b/ompi/mca/pml/bfo/pml_bfo_rdmafrag.c deleted file mode 100644 index b99e30a8de..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_rdmafrag.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "pml_bfo.h" -#include "pml_bfo_rdmafrag.h" - - -OBJ_CLASS_INSTANCE( - mca_pml_bfo_rdma_frag_t, - ompi_free_list_item_t, - NULL, - NULL); diff --git a/ompi/mca/pml/bfo/pml_bfo_rdmafrag.h b/ompi/mca/pml/bfo/pml_bfo_rdmafrag.h deleted file mode 100644 index 883c16baa1..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_rdmafrag.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ - -#ifndef MCA_PML_BFO_RDMAFRAG_H -#define MCA_PML_BFO_RDMAFRAG_H - -#include "pml_bfo_hdr.h" -#include "opal/mca/mpool/base/base.h" - -BEGIN_C_DECLS - -typedef enum { - MCA_PML_BFO_RDMA_PUT, - MCA_PML_BFO_RDMA_GET -} mca_pml_bfo_rdma_state_t; - -struct mca_pml_bfo_rdma_frag_t { - opal_free_list_item_t super; - mca_bml_base_btl_t* rdma_bml; -#if PML_BFO - mca_btl_base_module_t* rdma_btl; -#endif /* PML_BFO */ - mca_pml_bfo_hdr_t rdma_hdr; - mca_pml_bfo_rdma_state_t rdma_state; - size_t rdma_length; - uint8_t rdma_segs[MCA_BTL_SEG_MAX_SIZE * MCA_BTL_DES_MAX_SEGMENTS]; - void *rdma_req; - struct mca_bml_base_endpoint_t* rdma_ep; - opal_convertor_t convertor; - struct mca_mpool_base_registration_t* reg; - uint32_t retries; -}; -typedef struct mca_pml_bfo_rdma_frag_t mca_pml_bfo_rdma_frag_t; - -OBJ_CLASS_DECLARATION(mca_pml_bfo_rdma_frag_t); - - -#define MCA_PML_BFO_RDMA_FRAG_ALLOC(frag) \ -do { \ - opal_free_list_item_t* item; \ - OPAL_FREE_LIST_WAIT_MT(&mca_pml_bfo.rdma_frags, item); \ - frag = (mca_pml_bfo_rdma_frag_t*)item; \ -} while(0) - -#define MCA_PML_BFO_RDMA_FRAG_RETURN(frag) \ -do { \ - /* return fragment */ \ - OPAL_FREE_LIST_RETURN_MT(&mca_pml_bfo.rdma_frags, \ - (opal_free_list_item_t*)frag); \ -} while(0) - - -END_C_DECLS - -#endif - diff --git a/ompi/mca/pml/bfo/pml_bfo_recvfrag.c b/ompi/mca/pml/bfo/pml_bfo_recvfrag.c deleted file mode 100644 index c7216c0d53..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_recvfrag.c +++ /dev/null @@ -1,743 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2006-2008 University of Houston. All rights reserved. - * Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/** - * @file - */ - -#include "ompi_config.h" - -#include "opal/class/opal_list.h" -#include "opal/threads/mutex.h" -#include "opal/prefetch.h" - -#include "ompi/constants.h" -#include "ompi/communicator/communicator.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/peruse/peruse-internal.h" -#include "ompi/memchecker.h" - -#include "pml_bfo.h" -#include "pml_bfo_comm.h" -#include "pml_bfo_recvfrag.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_hdr.h" -#if PML_BFO -#include "pml_bfo_failover.h" -#endif /* PML_BFO */ - -OBJ_CLASS_INSTANCE( mca_pml_bfo_buffer_t, - ompi_free_list_item_t, - NULL, - NULL ); - -OBJ_CLASS_INSTANCE( mca_pml_bfo_recv_frag_t, - opal_list_item_t, - NULL, - NULL ); - -/** - * Static functions. - */ - -/** - * Append a unexpected descriptor to a queue. This function will allocate and - * initialize the fragment (if necessary) and then will add it to the specified - * queue. The allocated fragment is not returned to the caller. - */ -static void -append_frag_to_list(opal_list_t *queue, mca_btl_base_module_t *btl, - mca_pml_bfo_match_hdr_t *hdr, mca_btl_base_segment_t* segments, - size_t num_segments, mca_pml_bfo_recv_frag_t* frag) -{ - if(NULL == frag) { - MCA_PML_BFO_RECV_FRAG_ALLOC(frag); - MCA_PML_BFO_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl); - } - opal_list_append(queue, (opal_list_item_t*)frag); -} - -/** - * Match incoming recv_frags against posted receives. - * Supports out of order delivery. - * - * @param frag_header (IN) Header of received recv_frag. - * @param frag_desc (IN) Received recv_frag descriptor. - * @param match_made (OUT) Flag indicating wether a match was made. - * @param additional_matches (OUT) List of additional matches - * @return OMPI_SUCCESS or error status on failure. - */ -static int mca_pml_bfo_recv_frag_match( mca_btl_base_module_t *btl, - mca_pml_bfo_match_hdr_t *hdr, - mca_btl_base_segment_t* segments, - size_t num_segments, - int type); - -static mca_pml_bfo_recv_request_t* -match_one(mca_btl_base_module_t *btl, - mca_pml_bfo_match_hdr_t *hdr, mca_btl_base_segment_t* segments, - size_t num_segments, ompi_communicator_t *comm_ptr, - mca_pml_bfo_comm_proc_t *proc, - mca_pml_bfo_recv_frag_t* frag); - -void mca_pml_bfo_recv_frag_callback_match(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) -{ - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_match_hdr_t* hdr = (mca_pml_bfo_match_hdr_t*)segments->seg_addr.pval; - ompi_communicator_t *comm_ptr; - mca_pml_bfo_recv_request_t *match = NULL; - mca_pml_bfo_comm_t *comm; - mca_pml_bfo_comm_proc_t *proc; - size_t num_segments = des->des_local_count; - size_t bytes_received = 0; - - assert(num_segments <= MCA_BTL_DES_MAX_SEGMENTS); - - if( OPAL_UNLIKELY(segments->seg_len < OMPI_PML_BFO_MATCH_HDR_LEN) ) { - return; - } - bfo_hdr_ntoh(((mca_pml_bfo_hdr_t*) hdr), MCA_PML_BFO_HDR_TYPE_MATCH); - - /* communicator pointer */ - comm_ptr = ompi_comm_lookup(hdr->hdr_ctx); - if(OPAL_UNLIKELY(NULL == comm_ptr)) { - /* This is a special case. A message for a not yet existing - * communicator can happens. Instead of doing a matching we - * will temporarily add it the a pending queue in the PML. - * Later on, when the communicator is completely instantiated, - * this pending queue will be searched and all matching fragments - * moved to the right communicator. - */ - append_frag_to_list( &mca_pml_bfo.non_existing_communicator_pending, - btl, hdr, segments, num_segments, NULL ); - return; - } - comm = (mca_pml_bfo_comm_t *)comm_ptr->c_pml_comm; - - /* source sequence number */ - proc = &comm->procs[hdr->hdr_src]; - - /* We generate the MSG_ARRIVED event as soon as the PML is aware - * of a matching fragment arrival. Independing if it is received - * on the correct order or not. This will allow the tools to - * figure out if the messages are not received in the correct - * order (if multiple network interfaces). - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - /* get next expected message sequence number - if threaded - * run, lock to make sure that if another thread is processing - * a frag from the same message a match is made only once. - * Also, this prevents other posted receives (for a pair of - * end points) from being processed, and potentially "loosing" - * the fragment. - */ - OPAL_THREAD_LOCK(&comm->matching_lock); - - /* get sequence number of next message that can be processed */ - if(OPAL_UNLIKELY((((uint16_t) hdr->hdr_seq) != ((uint16_t) proc->expected_sequence)) || - (opal_list_get_size(&proc->frags_cant_match) > 0 ))) { - goto slow_path; - } - - /* This is the sequence number we were expecting, so we can try - * matching it to already posted receives. - */ - - /* We're now expecting the next sequence number. */ - proc->expected_sequence++; - - /* We generate the SEARCH_POSTED_QUEUE only when the message is - * received in the correct sequence. Otherwise, we delay the event - * generation until we reach the correct sequence number. - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_BEGIN, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - match = match_one(btl, hdr, segments, num_segments, comm_ptr, proc, NULL); - - /* The match is over. We generate the SEARCH_POSTED_Q_END here, - * before going into the mca_pml_bfo_check_cantmatch_for_match so - * we can make a difference for the searching time for all - * messages. - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_END, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - /* release matching lock before processing fragment */ - OPAL_THREAD_UNLOCK(&comm->matching_lock); - - if(OPAL_LIKELY(match)) { - bytes_received = segments->seg_len - OMPI_PML_BFO_MATCH_HDR_LEN; - match->req_recv.req_bytes_packed = bytes_received; - - MCA_PML_BFO_RECV_REQUEST_MATCHED(match, hdr); - if(match->req_bytes_expected > 0) { - struct iovec iov[MCA_BTL_DES_MAX_SEGMENTS]; - uint32_t iov_count = 1; - - /* - * Make user buffer accessable(defined) before unpacking. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - match->req_recv.req_base.req_addr, - match->req_recv.req_base.req_count, - match->req_recv.req_base.req_datatype); - ); - - iov[0].iov_len = bytes_received; - iov[0].iov_base = (IOVBASE_TYPE*)((unsigned char*)segments->seg_addr.pval + - OMPI_PML_BFO_MATCH_HDR_LEN); - while (iov_count < num_segments) { - bytes_received += segments[iov_count].seg_len; - iov[iov_count].iov_len = segments[iov_count].seg_len; - iov[iov_count].iov_base = (IOVBASE_TYPE*)((unsigned char*)segments[iov_count].seg_addr.pval); - iov_count++; - } - opal_convertor_unpack( &match->req_recv.req_base.req_convertor, - iov, - &iov_count, - &bytes_received ); - match->req_bytes_received = bytes_received; - /* - * Unpacking finished, make the user buffer unaccessable again. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - match->req_recv.req_base.req_addr, - match->req_recv.req_base.req_count, - match->req_recv.req_base.req_datatype); - ); - } - - /* no need to check if complete we know we are.. */ - /* don't need a rmb as that is for checking */ - recv_request_pml_complete(match); - } - return; - - slow_path: - OPAL_THREAD_UNLOCK(&comm->matching_lock); -#if PML_BFO - if (true == mca_pml_bfo_is_duplicate_msg(proc, hdr)) { - return; - } -#endif /* PML_BFO */ - mca_pml_bfo_recv_frag_match(btl, hdr, segments, - num_segments, MCA_PML_BFO_HDR_TYPE_MATCH); -} - - -void mca_pml_bfo_recv_frag_callback_rndv(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) -{ - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - - if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_bfo_common_hdr_t)) ) { - return; - } - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_RNDV); - mca_pml_bfo_recv_frag_match(btl, &hdr->hdr_match, segments, - des->des_local_count, MCA_PML_BFO_HDR_TYPE_RNDV); - return; -} - -void mca_pml_bfo_recv_frag_callback_rget(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) -{ - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - - if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_bfo_common_hdr_t)) ) { - return; - } - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_RGET); - mca_pml_bfo_recv_frag_match(btl, &hdr->hdr_match, segments, - des->des_local_count, MCA_PML_BFO_HDR_TYPE_RGET); - return; -} - - - -void mca_pml_bfo_recv_frag_callback_ack(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) -{ - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_send_request_t* sendreq; - - if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_bfo_common_hdr_t)) ) { - return; - } - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_ACK); - sendreq = (mca_pml_bfo_send_request_t*)hdr->hdr_ack.hdr_src_req.pval; - sendreq->req_recv = hdr->hdr_ack.hdr_dst_req; -#if PML_BFO - MCA_PML_BFO_ERROR_CHECK_ON_ACK_CALLBACK(sendreq); -#endif /* PML_BFO */ - - /* if the request should be delivered entirely by copy in/out - * then throttle sends */ - if(hdr->hdr_common.hdr_flags & MCA_PML_BFO_HDR_FLAGS_NORDMA) - sendreq->req_throttle_sends = true; - - mca_pml_bfo_send_request_copy_in_out(sendreq, - hdr->hdr_ack.hdr_send_offset, - sendreq->req_send.req_bytes_packed - - hdr->hdr_ack.hdr_send_offset); - - if (sendreq->req_state != 0) { - /* Typical receipt of an ACK message causes req_state to be - * decremented. However, a send request that started as an - * RGET request can become a RNDV. For example, when the - * receiver determines that its receive buffer is not - * contiguous and therefore cannot support the RGET - * protocol. A send request that started with the RGET - * protocol has req_state == 0 and as such should not be - * decremented. - */ - OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, -1); - } - - if(send_request_pml_complete_check(sendreq) == false) - mca_pml_bfo_send_request_schedule(sendreq); - - return; -} - -void mca_pml_bfo_recv_frag_callback_frag(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_recv_request_t* recvreq; - - if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_bfo_common_hdr_t)) ) { - return; - } - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_FRAG); - recvreq = (mca_pml_bfo_recv_request_t*)hdr->hdr_frag.hdr_dst_req.pval; -#if PML_BFO - MCA_PML_BFO_ERROR_CHECK_ON_FRAG_CALLBACK(recvreq); -#endif /* PML_BFO */ - mca_pml_bfo_recv_request_progress_frag(recvreq,btl,segments,des->des_local_count); - - return; -} - - -void mca_pml_bfo_recv_frag_callback_put(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_pml_bfo_send_request_t* sendreq; - - if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_bfo_common_hdr_t)) ) { - return; - } - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_PUT); - sendreq = (mca_pml_bfo_send_request_t*)hdr->hdr_rdma.hdr_req.pval; -#if PML_BFO - MCA_PML_BFO_ERROR_CHECK_ON_PUT_CALLBACK(sendreq); -#endif /* PML_BFO */ - mca_pml_bfo_send_request_put(sendreq,btl,&hdr->hdr_rdma); - - return; -} - - -void mca_pml_bfo_recv_frag_callback_fin(mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* des, - void* cbdata ) { - mca_btl_base_segment_t* segments = des->des_local; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - mca_btl_base_descriptor_t* rdma; - - if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_bfo_common_hdr_t)) ) { - return; - } - - bfo_hdr_ntoh(hdr, MCA_PML_BFO_HDR_TYPE_FIN); - rdma = (mca_btl_base_descriptor_t*)hdr->hdr_fin.hdr_des.pval; -#if PML_BFO - if (true == mca_pml_bfo_is_duplicate_fin(hdr, rdma, btl)) { - return; - } -#endif /* PML_BFO */ - rdma->des_cbfunc(btl, NULL, rdma, - hdr->hdr_fin.hdr_fail ? OMPI_ERROR : OMPI_SUCCESS); - - return; -} - - - -#define PML_MAX_SEQ ~((mca_pml_sequence_t)0); - -static inline mca_pml_bfo_recv_request_t* get_posted_recv(opal_list_t *queue) -{ - if(opal_list_get_size(queue) == 0) - return NULL; - - return (mca_pml_bfo_recv_request_t*)opal_list_get_first(queue); -} - -static inline mca_pml_bfo_recv_request_t* get_next_posted_recv( - opal_list_t *queue, - mca_pml_bfo_recv_request_t* req) -{ - opal_list_item_t *i = opal_list_get_next((opal_list_item_t*)req); - - if(opal_list_get_end(queue) == i) - return NULL; - - return (mca_pml_bfo_recv_request_t*)i; -} - -static mca_pml_bfo_recv_request_t *match_incomming( - mca_pml_bfo_match_hdr_t *hdr, mca_pml_bfo_comm_t *comm, - mca_pml_bfo_comm_proc_t *proc) -{ - mca_pml_bfo_recv_request_t *specific_recv, *wild_recv; - mca_pml_sequence_t wild_recv_seq, specific_recv_seq; - int tag = hdr->hdr_tag; - - specific_recv = get_posted_recv(&proc->specific_receives); - wild_recv = get_posted_recv(&comm->wild_receives); - - wild_recv_seq = wild_recv ? - wild_recv->req_recv.req_base.req_sequence : PML_MAX_SEQ; - specific_recv_seq = specific_recv ? - specific_recv->req_recv.req_base.req_sequence : PML_MAX_SEQ; - - /* they are equal only if both are PML_MAX_SEQ */ - while(wild_recv_seq != specific_recv_seq) { - mca_pml_bfo_recv_request_t **match; - opal_list_t *queue; - int req_tag; - mca_pml_sequence_t *seq; - - if (OPAL_UNLIKELY(wild_recv_seq < specific_recv_seq)) { - match = &wild_recv; - queue = &comm->wild_receives; - seq = &wild_recv_seq; - } else { - match = &specific_recv; - queue = &proc->specific_receives; - seq = &specific_recv_seq; - } - - req_tag = (*match)->req_recv.req_base.req_tag; - if(req_tag == tag || (req_tag == OMPI_ANY_TAG && tag >= 0)) { - opal_list_remove_item(queue, (opal_list_item_t*)(*match)); - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_REQ_REMOVE_FROM_POSTED_Q, - &((*match)->req_recv.req_base), PERUSE_RECV); - return *match; - } - - *match = get_next_posted_recv(queue, *match); - *seq = (*match) ? (*match)->req_recv.req_base.req_sequence : PML_MAX_SEQ; - } - - return NULL; -} - -static mca_pml_bfo_recv_request_t* -match_one(mca_btl_base_module_t *btl, - mca_pml_bfo_match_hdr_t *hdr, mca_btl_base_segment_t* segments, - size_t num_segments, ompi_communicator_t *comm_ptr, - mca_pml_bfo_comm_proc_t *proc, - mca_pml_bfo_recv_frag_t* frag) -{ - mca_pml_bfo_recv_request_t *match; - mca_pml_bfo_comm_t *comm = (mca_pml_bfo_comm_t *)comm_ptr->c_pml_comm; - - do { - match = match_incomming(hdr, comm, proc); - - /* if match found, process data */ - if(OPAL_LIKELY(NULL != match)) { - match->req_recv.req_base.req_proc = proc->ompi_proc; - - if(OPAL_UNLIKELY(MCA_PML_REQUEST_PROBE == match->req_recv.req_base.req_type)) { - /* complete the probe */ - mca_pml_bfo_recv_request_matched_probe(match, btl, segments, - num_segments); - /* attempt to match actual request */ - continue; - } else if (MCA_PML_REQUEST_MPROBE == match->req_recv.req_base.req_type) { - /* create a receive frag and associate it with the - request, which is then completed so that it can be - restarted later during mrecv */ - mca_pml_bfo_recv_frag_t *tmp; - if(NULL == frag) { - MCA_PML_BFO_RECV_FRAG_ALLOC(tmp); - MCA_PML_BFO_RECV_FRAG_INIT(tmp, hdr, segments, num_segments, btl); - } else { - tmp = frag; - } - - match->req_recv.req_base.req_addr = tmp; - mca_pml_bfo_recv_request_matched_probe(match, btl, segments, - num_segments); - /* this frag is already processed, so we want to break out - of the loop and not end up back on the unexpected queue. */ - return NULL; - } - - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_MSG_MATCH_POSTED_REQ, - &(match->req_recv.req_base), PERUSE_RECV); - return match; - } - - /* if no match found, place on unexpected queue */ - append_frag_to_list(&proc->unexpected_frags, btl, hdr, segments, - num_segments, frag); - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - return NULL; - } while(true); -} - -static mca_pml_bfo_recv_frag_t* check_cantmatch_for_match(mca_pml_bfo_comm_proc_t *proc) -{ - mca_pml_bfo_recv_frag_t *frag; - - /* search the list for a fragment from the send with sequence - * number next_msg_seq_expected - */ - for(frag = (mca_pml_bfo_recv_frag_t*)opal_list_get_first(&proc->frags_cant_match); - frag != (mca_pml_bfo_recv_frag_t*)opal_list_get_end(&proc->frags_cant_match); - frag = (mca_pml_bfo_recv_frag_t*)opal_list_get_next(frag)) - { - mca_pml_bfo_match_hdr_t* hdr = &frag->hdr.hdr_match; - /* - * If the message has the next expected seq from that proc... - */ - if(hdr->hdr_seq != proc->expected_sequence) - continue; - - opal_list_remove_item(&proc->frags_cant_match, (opal_list_item_t*)frag); - return frag; - } - - return NULL; -} - -/** - * RCS/CTS receive side matching - * - * @param hdr list of parameters needed for matching - * This list is also embeded in frag, - * but this allows to save a memory copy when - * a match is made in this routine. (IN) - * @param frag pointer to receive fragment which we want - * to match (IN/OUT). If a match is not made, - * hdr is copied to frag. - * @param match_made parameter indicating if we matched frag/ - * hdr (OUT) - * @param additional_matches if a match is made with frag, we - * may be able to match fragments that previously - * have arrived out-of-order. If this is the - * case, the associated fragment descriptors are - * put on this list for further processing. (OUT) - * - * @return OMPI error code - * - * This routine is used to try and match a newly arrived message fragment - * to pre-posted receives. The following assumptions are made - * - fragments are received out of order - * - for long messages, e.g. more than one fragment, a RTS/CTS algorithm - * is used. - * - 2nd and greater fragments include a receive descriptor pointer - * - fragments may be dropped - * - fragments may be corrupt - * - this routine may be called simultaneously by more than one thread - */ -static int mca_pml_bfo_recv_frag_match( mca_btl_base_module_t *btl, - mca_pml_bfo_match_hdr_t *hdr, - mca_btl_base_segment_t* segments, - size_t num_segments, - int type) -{ - /* local variables */ - uint16_t next_msg_seq_expected, frag_msg_seq; - ompi_communicator_t *comm_ptr; - mca_pml_bfo_recv_request_t *match = NULL; - mca_pml_bfo_comm_t *comm; - mca_pml_bfo_comm_proc_t *proc; - mca_pml_bfo_recv_frag_t* frag = NULL; - - /* communicator pointer */ - comm_ptr = ompi_comm_lookup(hdr->hdr_ctx); - if(OPAL_UNLIKELY(NULL == comm_ptr)) { - /* This is a special case. A message for a not yet existing - * communicator can happens. Instead of doing a matching we - * will temporarily add it the a pending queue in the PML. - * Later on, when the communicator is completely instantiated, - * this pending queue will be searched and all matching fragments - * moved to the right communicator. - */ - append_frag_to_list( &mca_pml_bfo.non_existing_communicator_pending, - btl, hdr, segments, num_segments, NULL ); - return OMPI_SUCCESS; - } - comm = (mca_pml_bfo_comm_t *)comm_ptr->c_pml_comm; - - /* source sequence number */ - frag_msg_seq = hdr->hdr_seq; - proc = &comm->procs[hdr->hdr_src]; - - /** - * We generate the MSG_ARRIVED event as soon as the PML is aware of a matching - * fragment arrival. Independing if it is received on the correct order or not. - * This will allow the tools to figure out if the messages are not received in the - * correct order (if multiple network interfaces). - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - /* get next expected message sequence number - if threaded - * run, lock to make sure that if another thread is processing - * a frag from the same message a match is made only once. - * Also, this prevents other posted receives (for a pair of - * end points) from being processed, and potentially "loosing" - * the fragment. - */ - OPAL_THREAD_LOCK(&comm->matching_lock); - -#if PML_BFO - if(OPAL_UNLIKELY(hdr->hdr_common.hdr_flags & MCA_PML_BFO_HDR_FLAGS_RESTART)) { - if (NULL == (match = mca_pml_bfo_get_request(hdr))) { - return OMPI_SUCCESS; - } - } else { -#endif /* PML_BFO */ - /* get sequence number of next message that can be processed */ - next_msg_seq_expected = (uint16_t)proc->expected_sequence; - if(OPAL_UNLIKELY(frag_msg_seq != next_msg_seq_expected)) - goto wrong_seq; - - /* - * This is the sequence number we were expecting, - * so we can try matching it to already posted - * receives. - */ - -out_of_order_match: - /* We're now expecting the next sequence number. */ - proc->expected_sequence++; - - /** - * We generate the SEARCH_POSTED_QUEUE only when the message is received - * in the correct sequence. Otherwise, we delay the event generation until - * we reach the correct sequence number. - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_BEGIN, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - match = match_one(btl, hdr, segments, num_segments, comm_ptr, proc, frag); - - /** - * The match is over. We generate the SEARCH_POSTED_Q_END here, before going - * into the mca_pml_bfo_check_cantmatch_for_match so we can make a difference - * for the searching time for all messages. - */ - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_END, comm_ptr, - hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); - - /* release matching lock before processing fragment */ - OPAL_THREAD_UNLOCK(&comm->matching_lock); - -#if PML_BFO - } -#endif /* PML_BFO */ - if(OPAL_LIKELY(match)) { - switch(type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - mca_pml_bfo_recv_request_progress_match(match, btl, segments, num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - mca_pml_bfo_recv_request_progress_rndv(match, btl, segments, num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - mca_pml_bfo_recv_request_progress_rget(match, btl, segments, num_segments); - break; - } - - if(OPAL_UNLIKELY(frag)) - MCA_PML_BFO_RECV_FRAG_RETURN(frag); - } - - /* - * Now that new message has arrived, check to see if - * any fragments on the c_c_frags_cant_match list - * may now be used to form new matchs - */ - if(OPAL_UNLIKELY(opal_list_get_size(&proc->frags_cant_match) > 0)) { - OPAL_THREAD_LOCK(&comm->matching_lock); - if((frag = check_cantmatch_for_match(proc))) { - hdr = &frag->hdr.hdr_match; - segments = frag->segments; - num_segments = frag->num_segments; - btl = frag->btl; - type = hdr->hdr_common.hdr_type; - goto out_of_order_match; - } - OPAL_THREAD_UNLOCK(&comm->matching_lock); - } - - return OMPI_SUCCESS; -wrong_seq: - /* - * This message comes after the next expected, so it - * is ahead of sequence. Save it for later. - */ -#if PML_BFO - if (true == mca_pml_bfo_is_duplicate_msg(proc, hdr)) { - return OMPI_SUCCESS; - } -#endif /* PML_BFO */ - append_frag_to_list(&proc->frags_cant_match, btl, hdr, segments, - num_segments, NULL); - OPAL_THREAD_UNLOCK(&comm->matching_lock); - return OMPI_SUCCESS; -} - diff --git a/ompi/mca/pml/bfo/pml_bfo_recvfrag.h b/ompi/mca/pml/bfo/pml_bfo_recvfrag.h deleted file mode 100644 index 72c557daac..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_recvfrag.h +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ - -#ifndef MCA_PML_BFO_RECVFRAG_H -#define MCA_PML_BFO_RECVFRAG_H - -#include "pml_bfo_hdr.h" - -BEGIN_C_DECLS - -struct mca_pml_bfo_buffer_t { - size_t len; - void * addr; -}; -typedef struct mca_pml_bfo_buffer_t mca_pml_bfo_buffer_t; - - -struct mca_pml_bfo_recv_frag_t { - opal_free_list_item_t super; - mca_pml_bfo_hdr_t hdr; - size_t num_segments; - mca_btl_base_module_t* btl; - mca_btl_base_segment_t segments[MCA_BTL_DES_MAX_SEGMENTS]; - mca_pml_bfo_buffer_t buffers[MCA_BTL_DES_MAX_SEGMENTS]; - unsigned char addr[1]; -}; -typedef struct mca_pml_bfo_recv_frag_t mca_pml_bfo_recv_frag_t; - -OBJ_CLASS_DECLARATION(mca_pml_bfo_recv_frag_t); - - -#define MCA_PML_BFO_RECV_FRAG_ALLOC(frag) \ -do { \ - opal_free_list_item_t* item; \ - OPAL_FREE_LIST_WAIT_MT(&mca_pml_bfo.recv_frags, item); \ - frag = (mca_pml_bfo_recv_frag_t*)item; \ -} while(0) - - -#define MCA_PML_BFO_RECV_FRAG_INIT(frag, hdr, segs, cnt, btl ) \ -do { \ - size_t i, _size; \ - mca_btl_base_segment_t* macro_segments = frag->segments; \ - mca_pml_bfo_buffer_t* buffers = frag->buffers; \ - unsigned char* _ptr = (unsigned char*)frag->addr; \ - /* init recv_frag */ \ - frag->btl = btl; \ - frag->hdr = *(mca_pml_bfo_hdr_t*)hdr; \ - frag->num_segments = 1; \ - _size = segs[0].seg_len; \ - for( i = 1; i < cnt; i++ ) { \ - _size += segs[i].seg_len; \ - } \ - /* copy over data */ \ - if(_size <= mca_pml_bfo.unexpected_limit ) { \ - macro_segments[0].seg_addr.pval = frag->addr; \ - } else { \ - buffers[0].len = _size; \ - buffers[0].addr = (char*) \ - mca_pml_bfo.allocator->alc_alloc( mca_pml_bfo.allocator, \ - buffers[0].len, \ - 0, NULL); \ - _ptr = (unsigned char*)(buffers[0].addr); \ - macro_segments[0].seg_addr.pval = buffers[0].addr; \ - } \ - macro_segments[0].seg_len = _size; \ - for( i = 0; i < cnt; i++ ) { \ - memcpy( _ptr, segs[i].seg_addr.pval, segs[i].seg_len); \ - _ptr += segs[i].seg_len; \ - } \ - } while(0) - - -#define MCA_PML_BFO_RECV_FRAG_RETURN(frag) \ -do { \ - if( frag->segments[0].seg_len > mca_pml_bfo.unexpected_limit ) { \ - /* return buffers */ \ - mca_pml_bfo.allocator->alc_free( mca_pml_bfo.allocator, \ - frag->buffers[0].addr ); \ - } \ - frag->num_segments = 0; \ - \ - /* return recv_frag */ \ - OPAL_FREE_LIST_RETURN(&mca_pml_bfo.recv_frags, \ - (opal_free_list_item_t*)frag); \ - } while(0) - - -/** - * Callback from BTL on receipt of a recv_frag (match). - */ - -extern void mca_pml_bfo_recv_frag_callback_match( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -/** - * Callback from BTL on receipt of a recv_frag (rndv). - */ - -extern void mca_pml_bfo_recv_frag_callback_rndv( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); -/** - * Callback from BTL on receipt of a recv_frag (rget). - */ - -extern void mca_pml_bfo_recv_frag_callback_rget( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - -/** - * Callback from BTL on receipt of a recv_frag (ack). - */ - -extern void mca_pml_bfo_recv_frag_callback_ack( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); -/** - * Callback from BTL on receipt of a recv_frag (frag). - */ - -extern void mca_pml_bfo_recv_frag_callback_frag( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); -/** - * Callback from BTL on receipt of a recv_frag (put). - */ - -extern void mca_pml_bfo_recv_frag_callback_put( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); -/** - * Callback from BTL on receipt of a recv_frag (fin). - */ - -extern void mca_pml_bfo_recv_frag_callback_fin( mca_btl_base_module_t *btl, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t* descriptor, - void* cbdata ); - - -END_C_DECLS - -#endif - diff --git a/ompi/mca/pml/bfo/pml_bfo_recvreq.c b/ompi/mca/pml/bfo/pml_bfo_recvreq.c deleted file mode 100644 index c0658f10ef..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_recvreq.c +++ /dev/null @@ -1,1165 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011-2012 Los Alamos National Security, LLC. - * All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "opal/mca/mpool/mpool.h" -#include "opal/util/arch.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/bml/bml.h" -#include "pml_bfo_comm.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_recvfrag.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_rdmafrag.h" -#include "ompi/mca/bml/base/base.h" -#include "ompi/memchecker.h" -#if OPAL_CUDA_SUPPORT -#include "opal/datatype/opal_datatype_cuda.h" -#include "opal/mca/common/cuda/common_cuda.h" -#endif /* OPAL_CUDA_SUPPORT */ - -#if OPAL_CUDA_SUPPORT -int mca_pml_bfo_cuda_need_buffers(mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_module_t* btl); -#endif /* OPAL_CUDA_SUPPORT */ -#if PML_BFO -#include "pml_bfo_failover.h" -#endif /* PML_BFO */ - -void mca_pml_bfo_recv_request_process_pending(void) -{ - mca_pml_bfo_recv_request_t* recvreq; - int rc, i, s = (int)opal_list_get_size(&mca_pml_bfo.recv_pending); - - for(i = 0; i < s; i++) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - recvreq = (mca_pml_bfo_recv_request_t*) - opal_list_remove_first(&mca_pml_bfo.recv_pending); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - if( OPAL_UNLIKELY(NULL == recvreq) ) - break; - recvreq->req_pending = false; - rc = mca_pml_bfo_recv_request_schedule_exclusive(recvreq, NULL); - if(OMPI_ERR_OUT_OF_RESOURCE == rc) - break; - } -} - -static int mca_pml_bfo_recv_request_free(struct ompi_request_t** request) -{ - mca_pml_bfo_recv_request_t* recvreq = *(mca_pml_bfo_recv_request_t**)request; - - if(false == recvreq->req_recv.req_base.req_free_called) { - - recvreq->req_recv.req_base.req_free_called = true; - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_NOTIFY, - &(recvreq->req_recv.req_base), PERUSE_RECV ); - - if( true == recvreq->req_recv.req_base.req_pml_complete ) { - /* make buffer defined when the request is compeleted, - and before releasing the objects. */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - - MCA_PML_BFO_RECV_REQUEST_RETURN( recvreq ); - } - - } - *request = MPI_REQUEST_NULL; - return OMPI_SUCCESS; -} - -static int mca_pml_bfo_recv_request_cancel(struct ompi_request_t* ompi_request, int complete) -{ - mca_pml_bfo_recv_request_t* request = (mca_pml_bfo_recv_request_t*)ompi_request; - mca_pml_bfo_comm_t* comm = request->req_recv.req_base.req_comm->c_pml_comm; - - if( true == ompi_request->req_complete ) { /* way to late to cancel this one */ - /* - * Receive request completed, make user buffer accessable. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - request->req_recv.req_base.req_addr, - request->req_recv.req_base.req_count, - request->req_recv.req_base.req_datatype); - ); - return OMPI_SUCCESS; - } - - /* The rest should be protected behind the match logic lock */ - OPAL_THREAD_LOCK(&comm->matching_lock); - if( OMPI_ANY_TAG == ompi_request->req_status.MPI_TAG ) { /* the match has not been already done */ - if( request->req_recv.req_base.req_peer == OMPI_ANY_SOURCE ) { - opal_list_remove_item( &comm->wild_receives, (opal_list_item_t*)request ); - } else { - mca_pml_bfo_comm_proc_t* proc = comm->procs + request->req_recv.req_base.req_peer; - opal_list_remove_item(&proc->specific_receives, (opal_list_item_t*)request); - } - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_REMOVE_FROM_POSTED_Q, - &(request->req_recv.req_base), PERUSE_RECV ); - /** - * As now the PML is done with this request we have to force the pml_complete - * to true. Otherwise, the request will never be freed. - */ - request->req_recv.req_base.req_pml_complete = true; - } - OPAL_THREAD_UNLOCK(&comm->matching_lock); - - ompi_request->req_status._cancelled = true; - /* This macro will set the req_complete to true so the MPI Test/Wait* functions - * on this request will be able to complete. As the status is marked as - * cancelled the cancel state will be detected. - */ - MCA_PML_BFO_RECV_REQUEST_MPI_COMPLETE(request); - /* - * Receive request cancelled, make user buffer accessable. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - request->req_recv.req_base.req_addr, - request->req_recv.req_base.req_count, - request->req_recv.req_base.req_datatype); - ); - return OMPI_SUCCESS; -} - -static void mca_pml_bfo_recv_request_construct(mca_pml_bfo_recv_request_t* request) -{ - request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV; - request->req_recv.req_base.req_ompi.req_start = mca_pml_bfo_start; - request->req_recv.req_base.req_ompi.req_free = mca_pml_bfo_recv_request_free; - request->req_recv.req_base.req_ompi.req_cancel = mca_pml_bfo_recv_request_cancel; - request->req_rdma_cnt = 0; - OBJ_CONSTRUCT(&request->lock, opal_mutex_t); -} - -OBJ_CLASS_INSTANCE( - mca_pml_bfo_recv_request_t, - mca_pml_base_recv_request_t, - mca_pml_bfo_recv_request_construct, - NULL); - - -/* - * Release resources. - */ - -static void mca_pml_bfo_recv_ctl_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*)des->des_context; - -#if PML_BFO - if (btl->btl_flags & MCA_BTL_FLAGS_FAILOVER_SUPPORT) { - mca_pml_bfo_check_recv_ctl_completion_status(btl, des, status); - } - MCA_PML_BFO_CHECK_RECVREQ_EAGER_BML_BTL_RECV_CTL(bml_btl, btl, des); -#endif /* PML_BFO */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -/* - * Put operation has completed remotely - update request status - */ - -static void mca_pml_bfo_put_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*)des->des_context; - mca_pml_bfo_recv_request_t* recvreq = (mca_pml_bfo_recv_request_t*)des->des_cbdata; - size_t bytes_received = 0; - - if( OPAL_LIKELY(status == OMPI_SUCCESS) ) { - bytes_received = mca_pml_bfo_compute_segment_length (btl->btl_seg_size, - (void *) des->des_remote, - des->des_remote_count, 0); - } - OPAL_THREAD_SUB_FETCH_SIZE_T(&recvreq->req_pipeline_depth, 1); - -#if PML_BFO - btl->btl_free(btl, des); - MCA_PML_BFO_ERROR_CHECK_ON_FIN_FOR_PUT(recvreq); - MCA_PML_BFO_CHECK_RECVREQ_EAGER_BML_BTL(bml_btl, btl, recvreq, "PUT"); -#else /* PML_BFO */ - mca_bml_base_free(bml_btl, des); -#endif /* PML_BFO */ - - /* check completion status */ - OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, bytes_received); - if(recv_request_pml_complete_check(recvreq) == false && - recvreq->req_rdma_offset < recvreq->req_send_offset) { - /* schedule additional rdma operations */ - mca_pml_bfo_recv_request_schedule(recvreq, bml_btl); - } - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -/* - * - */ - -int mca_pml_bfo_recv_request_ack_send_btl( - ompi_proc_t* proc, mca_bml_base_btl_t* bml_btl, - uint64_t hdr_src_req, void *hdr_dst_req, uint64_t hdr_send_offset, - bool nordma) -{ - mca_btl_base_descriptor_t* des; - mca_pml_bfo_ack_hdr_t* ack; - int rc; - - /* allocate descriptor */ - mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_ack_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK); - if( OPAL_UNLIKELY(NULL == des) ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* fill out header */ - ack = (mca_pml_bfo_ack_hdr_t*)des->des_local->seg_addr.pval; - ack->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_ACK; - ack->hdr_common.hdr_flags = nordma ? MCA_PML_BFO_HDR_FLAGS_NORDMA : 0; - ack->hdr_src_req.lval = hdr_src_req; - ack->hdr_dst_req.pval = hdr_dst_req; - ack->hdr_send_offset = hdr_send_offset; - - bfo_hdr_hton(ack, MCA_PML_BFO_HDR_TYPE_ACK, proc); - - /* initialize descriptor */ - des->des_cbfunc = mca_pml_bfo_recv_ctl_completion; -#if PML_BFO - des->des_cbdata = hdr_dst_req; -#endif /* PML_BFO */ - - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_ACK); - if( OPAL_LIKELY( rc >= 0 ) ) { -#if PML_BFO - if ((bml_btl->btl_flags & MCA_BTL_FLAGS_FAILOVER_SUPPORT) && - (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) { - ((mca_pml_bfo_recv_request_t *)hdr_dst_req)->req_events++; - } -#endif /* PML_BFO */ - return OMPI_SUCCESS; - } - mca_bml_base_free(bml_btl, des); - return OMPI_ERR_OUT_OF_RESOURCE; -} - -static int mca_pml_bfo_recv_request_ack( - mca_pml_bfo_recv_request_t* recvreq, - mca_pml_bfo_rendezvous_hdr_t* hdr, - size_t bytes_received) -{ - ompi_proc_t* proc = (ompi_proc_t*)recvreq->req_recv.req_base.req_proc; - mca_bml_base_endpoint_t* bml_endpoint = NULL; - - bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - /* by default copy everything */ - recvreq->req_send_offset = bytes_received; - if(hdr->hdr_msg_length > bytes_received) { - size_t rdma_num = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_rdma); - /* - * lookup request buffer to determine if memory is already - * registered. - */ - - if(opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == 0 && - hdr->hdr_match.hdr_common.hdr_flags & MCA_PML_BFO_HDR_FLAGS_CONTIG && - rdma_num != 0) { - unsigned char *base; - opal_convertor_get_current_pointer( &recvreq->req_recv.req_base.req_convertor, (void**)&(base) ); - - if(hdr->hdr_match.hdr_common.hdr_flags & MCA_PML_BFO_HDR_FLAGS_PIN) - recvreq->req_rdma_cnt = mca_pml_bfo_rdma_btls(bml_endpoint, - base, recvreq->req_recv.req_bytes_packed, - recvreq->req_rdma ); - else - recvreq->req_rdma_cnt = 0; - - /* memory is already registered on both sides */ - if (recvreq->req_rdma_cnt != 0) { - recvreq->req_send_offset = hdr->hdr_msg_length; - /* are rdma devices available for long rdma protocol */ - } else if(bml_endpoint->btl_send_limit < hdr->hdr_msg_length) { - /* use convertor to figure out the rdma offset for this request */ - recvreq->req_send_offset = hdr->hdr_msg_length - - bml_endpoint->btl_pipeline_send_length; - - if(recvreq->req_send_offset < bytes_received) - recvreq->req_send_offset = bytes_received; - - /* use converter to figure out the rdma offset for this - * request */ - opal_convertor_set_position(&recvreq->req_recv.req_base.req_convertor, - &recvreq->req_send_offset); - - recvreq->req_rdma_cnt = - mca_pml_bfo_rdma_pipeline_btls(bml_endpoint, - recvreq->req_send_offset - bytes_received, - recvreq->req_rdma); - } - } - /* nothing to send by copy in/out - no need to ack */ - if(recvreq->req_send_offset == hdr->hdr_msg_length) - return OMPI_SUCCESS; - } - /* let know to shedule function there is no need to put ACK flag */ - recvreq->req_ack_sent = true; - return mca_pml_bfo_recv_request_ack_send(proc, hdr->hdr_src_req.lval, - recvreq, recvreq->req_send_offset, - recvreq->req_send_offset == bytes_received); -} - -/** - * Return resources used by the RDMA - */ - -static void mca_pml_bfo_rget_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*)des->des_context; - mca_pml_bfo_rdma_frag_t* frag = (mca_pml_bfo_rdma_frag_t*)des->des_cbdata; - mca_pml_bfo_recv_request_t* recvreq = (mca_pml_bfo_recv_request_t*)frag->rdma_req; - -#if PML_BFO - if (btl->btl_flags & MCA_BTL_FLAGS_FAILOVER_SUPPORT) { - recvreq->req_events--; - } -#endif /* PML_BFO */ - /* check completion status */ - if( OPAL_UNLIKELY(OMPI_SUCCESS != status) ) { -#if PML_BFO - MCA_PML_BFO_ERROR_CHECK_ON_RDMA_READ_COMPLETION(recvreq); -#else /* PML_BFO */ - /* TSW - FIX */ - OMPI_ERROR_LOG(status); - ompi_rte_abort(-1, NULL); -#endif /* PML_BFO */ - } -#if PML_BFO - MCA_PML_BFO_SECOND_ERROR_CHECK_ON_RDMA_READ_COMPLETION(recvreq, status, btl); - MCA_PML_BFO_CHECK_RECVREQ_RDMA_BML_BTL(bml_btl, btl, recvreq, "RDMA write"); -#endif /* PML_BFO */ - - mca_pml_bfo_send_fin(recvreq->req_recv.req_base.req_proc, - bml_btl, - frag->rdma_hdr.hdr_rget.hdr_des, -#if PML_BFO - des->order, 0, (uint16_t)recvreq->req_msgseq, recvreq->req_restartseq, - recvreq->req_recv.req_base.req_comm->c_contextid, - recvreq->req_recv.req_base.req_comm->c_my_rank); -#else /* PML_BFO */ - des->order, 0); -#endif /* PML_BFO */ - - /* is receive request complete */ - OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, frag->rdma_length); - recv_request_pml_complete_check(recvreq); - - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); - - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - - -/* - * - */ -int mca_pml_bfo_recv_request_get_frag( mca_pml_bfo_rdma_frag_t* frag ) -{ - mca_pml_bfo_recv_request_t* recvreq = (mca_pml_bfo_recv_request_t*)frag->rdma_req; - mca_bml_base_btl_t* bml_btl = frag->rdma_bml; - mca_btl_base_descriptor_t* descriptor; - size_t save_size = frag->rdma_length; - int rc; - - /* prepare descriptor */ - mca_bml_base_prepare_dst( bml_btl, - NULL, - &recvreq->req_recv.req_base.req_convertor, - MCA_BTL_NO_ORDER, - 0, - &frag->rdma_length, - MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK | - MCA_BTL_DES_FLAGS_GET, - &descriptor ); - if( OPAL_UNLIKELY(NULL == descriptor) ) { - frag->rdma_length = save_size; - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.rdma_pending, (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - return OMPI_ERR_OUT_OF_RESOURCE; - } - - descriptor->des_remote = (mca_btl_base_segment_t *) frag->rdma_segs; - descriptor->des_remote_count = frag->rdma_hdr.hdr_rdma.hdr_seg_cnt; - descriptor->des_cbfunc = mca_pml_bfo_rget_completion; - descriptor->des_cbdata = frag; - - PERUSE_TRACE_COMM_OMPI_EVENT(PERUSE_COMM_REQ_XFER_CONTINUE, - &(recvreq->req_recv.req_base), - frag->rdma_length, PERUSE_RECV); - - /* queue up get request */ - rc = mca_bml_base_get(bml_btl,descriptor); - if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { - if(OMPI_ERR_OUT_OF_RESOURCE == rc) { - mca_bml_base_free(bml_btl, descriptor); - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.rdma_pending, - (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - return OMPI_ERR_OUT_OF_RESOURCE; - } else { - OMPI_ERROR_LOG(rc); - ompi_rte_abort(-1, NULL); - } - } -#if PML_BFO - if ((bml_btl->btl_flags & MCA_BTL_FLAGS_FAILOVER_SUPPORT) && - (descriptor->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) { - recvreq->req_events++; - } -#endif /* PML_BFO */ - - return OMPI_SUCCESS; -} - - - - -/* - * Update the recv request status to reflect the number of bytes - * received and actually delivered to the application. - */ - -void mca_pml_bfo_recv_request_progress_frag( mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments ) -{ - size_t bytes_received, data_offset = 0; - size_t bytes_delivered __opal_attribute_unused__; /* is being set to zero in MCA_PML_BFO_RECV_REQUEST_UNPACK */ - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - - bytes_received = mca_pml_bfo_compute_segment_length_base (segments, num_segments, - sizeof(mca_pml_bfo_frag_hdr_t)); - data_offset = hdr->hdr_frag.hdr_frag_offset; - /* - * Make user buffer accessable(defined) before unpacking. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - MCA_PML_BFO_RECV_REQUEST_UNPACK( recvreq, - segments, - num_segments, - sizeof(mca_pml_bfo_frag_hdr_t), - data_offset, - bytes_received, - bytes_delivered ); - /* - * Unpacking finished, make the user buffer unaccessable again. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - - OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, bytes_received); - /* check completion status */ - if(recv_request_pml_complete_check(recvreq) == false && - recvreq->req_rdma_offset < recvreq->req_send_offset) { - /* schedule additional rdma operations */ - mca_pml_bfo_recv_request_schedule(recvreq, NULL); - } -} - -/* - * Update the recv request status to reflect the number of bytes - * received and actually delivered to the application. - */ - -void mca_pml_bfo_recv_request_progress_rget( mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments ) -{ - mca_pml_bfo_rget_hdr_t* hdr = (mca_pml_bfo_rget_hdr_t*)segments->seg_addr.pval; - mca_bml_base_endpoint_t* bml_endpoint = NULL; - mca_pml_bfo_rdma_frag_t* frag; - size_t i, size = 0; - - recvreq->req_recv.req_bytes_packed = hdr->hdr_rndv.hdr_msg_length; - -#if PML_BFO - recvreq->remote_req_send = hdr->hdr_rndv.hdr_src_req; -#endif /* PML_BFO */ - MCA_PML_BFO_RECV_REQUEST_MATCHED(recvreq, &hdr->hdr_rndv.hdr_match); - - /* if receive buffer is not contiguous we can't just RDMA read into it, so - * fall back to copy in/out protocol. It is a pity because buffer on the - * sender side is already registered. We need to be smarter here, perhaps - * do couple of RDMA reads */ - if(opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == true) { -#if OPAL_CUDA_SUPPORT - if (mca_pml_bfo_cuda_need_buffers(recvreq, btl)) { - mca_pml_bfo_recv_request_ack(recvreq, &hdr->hdr_rndv, 0); - return; - } -#else /* OPAL_CUDA_SUPPORT */ - mca_pml_bfo_recv_request_ack(recvreq, &hdr->hdr_rndv, 0); - return; -#endif /* OPAL_CUDA_SUPPORT */ - } - - MCA_PML_BFO_RDMA_FRAG_ALLOC(frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - /* GLB - FIX */ - OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE); - ompi_rte_abort(-1, NULL); - } - - /* lookup bml datastructures */ - bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - assert (btl->btl_seg_size * hdr->hdr_seg_cnt <= sizeof (frag->rdma_segs)); - - /* allocate/initialize a fragment */ - memmove (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt); - - for(i = 0; i < hdr->hdr_seg_cnt; i++) { - mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *)(frag->rdma_segs + i * btl->btl_seg_size); - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - if ((recvreq->req_recv.req_base.req_proc->super.proc_arch & OPAL_ARCH_ISBIGENDIAN) != - (ompi_proc_local()->super.proc_arch & OPAL_ARCH_ISBIGENDIAN)) { - size += opal_swap_bytes4(seg->seg_len); - } else -#endif - { - size += seg->seg_len; - } - } -#if PML_BFO - frag->rdma_btl = btl; -#endif /* PML_BFO */ - frag->rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl); -#if OPAL_CUDA_SUPPORT - if( OPAL_UNLIKELY(NULL == frag->rdma_bml) ) { - if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) { - /* Check to see if this is a CUDA get */ - if (btl->btl_flags & MCA_BTL_FLAGS_CUDA_GET) { - frag->rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_send, btl); - } - if( OPAL_UNLIKELY(NULL == frag->rdma_bml) ) { - opal_output(0, "[%s:%d] invalid bml for rdma get", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } - } else { - /* Just default back to send and receive. Must be mix of GPU and HOST memory. */ - mca_pml_bfo_recv_request_ack(recvreq, &hdr->hdr_rndv, 0); - return; - } - } -#else /* OPAL_CUDA_SUPPORT */ - if( OPAL_UNLIKELY(NULL == frag->rdma_bml) ) { - opal_output(0, "[%s:%d] invalid bml for rdma get", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); - } -#endif /* OPAL_CUDA_SUPPORT */ - frag->rdma_hdr.hdr_rget = *hdr; - frag->rdma_req = recvreq; - frag->rdma_ep = bml_endpoint; - frag->rdma_length = size; - frag->rdma_state = MCA_PML_BFO_RDMA_GET; - frag->reg = NULL; - - mca_pml_bfo_recv_request_get_frag(frag); - return; -} - -/* - * Update the recv request status to reflect the number of bytes - * received and actually delivered to the application. - */ - -void mca_pml_bfo_recv_request_progress_rndv( mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments ) -{ - size_t bytes_received; - size_t bytes_delivered __opal_attribute_unused__; /* is being set to zero in MCA_PML_BFO_RECV_REQUEST_UNPACK */ - size_t data_offset = 0; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - - bytes_received = mca_pml_bfo_compute_segment_length_base (segments, num_segments, - sizeof(mca_pml_bfo_rendezvous_hdr_t)); - - recvreq->req_recv.req_bytes_packed = hdr->hdr_rndv.hdr_msg_length; - recvreq->remote_req_send = hdr->hdr_rndv.hdr_src_req; - recvreq->req_rdma_offset = bytes_received; - MCA_PML_BFO_RECV_REQUEST_MATCHED(recvreq, &hdr->hdr_match); - mca_pml_bfo_recv_request_ack(recvreq, &hdr->hdr_rndv, bytes_received); - /** - * The PUT protocol do not attach any data to the original request. - * Therefore, we might want to avoid unpacking if there is nothing to - * unpack. - */ - if( 0 < bytes_received ) { - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - MCA_PML_BFO_RECV_REQUEST_UNPACK( recvreq, - segments, - num_segments, - sizeof(mca_pml_bfo_rendezvous_hdr_t), - data_offset, - bytes_received, - bytes_delivered ); - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - } - OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, bytes_received); - /* check completion status */ - if(recv_request_pml_complete_check(recvreq) == false && - recvreq->req_rdma_offset < recvreq->req_send_offset) { - /* schedule additional rdma operations */ - mca_pml_bfo_recv_request_schedule(recvreq, NULL); - } -} - -/* - * Update the recv request status to reflect the number of bytes - * received and actually delivered to the application. - */ -void mca_pml_bfo_recv_request_progress_match( mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments ) -{ - size_t bytes_received, data_offset = 0; - size_t bytes_delivered __opal_attribute_unused__; /* is being set to zero in MCA_PML_BFO_RECV_REQUEST_UNPACK */ - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - - bytes_received = mca_pml_bfo_compute_segment_length_base (segments, num_segments, - OMPI_PML_BFO_MATCH_HDR_LEN); - - recvreq->req_recv.req_bytes_packed = bytes_received; - - MCA_PML_BFO_RECV_REQUEST_MATCHED(recvreq, &hdr->hdr_match); - /* - * Make user buffer accessable(defined) before unpacking. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - MCA_PML_BFO_RECV_REQUEST_UNPACK( recvreq, - segments, - num_segments, - OMPI_PML_BFO_MATCH_HDR_LEN, - data_offset, - bytes_received, - bytes_delivered); - /* - * Unpacking finished, make the user buffer unaccessable again. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - recvreq->req_recv.req_base.req_addr, - recvreq->req_recv.req_base.req_count, - recvreq->req_recv.req_base.req_datatype); - ); - - /* - * No need for atomic here, as we know there is only one fragment - * for this request. - */ - recvreq->req_bytes_received += bytes_received; - recv_request_pml_complete(recvreq); -} - - -/** - * Handle completion of a probe request - */ - -void mca_pml_bfo_recv_request_matched_probe( mca_pml_bfo_recv_request_t* recvreq, - mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments ) -{ - size_t bytes_packed = 0; - mca_pml_bfo_hdr_t* hdr = (mca_pml_bfo_hdr_t*)segments->seg_addr.pval; - - switch(hdr->hdr_common.hdr_type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - bytes_packed = mca_pml_bfo_compute_segment_length_base (segments, num_segments, - OMPI_PML_BFO_MATCH_HDR_LEN); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - case MCA_PML_BFO_HDR_TYPE_RGET: - bytes_packed = hdr->hdr_rndv.hdr_msg_length; - break; - } - - /* set completion status */ - recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_match.hdr_tag; - recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_match.hdr_src; - recvreq->req_bytes_received = bytes_packed; - recvreq->req_bytes_expected = bytes_packed; - - recv_request_pml_complete(recvreq); -} - - -/* - * Schedule RDMA protocol. - * -*/ - -int mca_pml_bfo_recv_request_schedule_once( mca_pml_bfo_recv_request_t* recvreq, - mca_bml_base_btl_t *start_bml_btl ) -{ - mca_bml_base_btl_t* bml_btl; - int num_tries = recvreq->req_rdma_cnt, num_fail = 0; - size_t i, prev_bytes_remaining = 0; - size_t bytes_remaining = recvreq->req_send_offset - - recvreq->req_rdma_offset; - - /* if starting bml_btl is provided schedule next fragment on it first */ - if(start_bml_btl != NULL) { - for(i = 0; i < recvreq->req_rdma_cnt; i++) { - if(recvreq->req_rdma[i].bml_btl != start_bml_btl) - continue; - /* something left to be send? */ - if( OPAL_LIKELY(recvreq->req_rdma[i].length) ) - recvreq->req_rdma_idx = i; - break; - } - } - - while(bytes_remaining > 0 && - recvreq->req_pipeline_depth < mca_pml_bfo.recv_pipeline_depth) { - size_t size, seg_size; - mca_pml_bfo_rdma_hdr_t* hdr; - mca_btl_base_descriptor_t* dst; - mca_btl_base_descriptor_t* ctl; - mca_mpool_base_registration_t * reg = NULL; - mca_btl_base_module_t* btl; - int rc, rdma_idx; - - if(prev_bytes_remaining == bytes_remaining) { - if(++num_fail == num_tries) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - if(false == recvreq->req_pending) { - opal_list_append(&mca_pml_bfo.recv_pending, - (opal_list_item_t*)recvreq); - recvreq->req_pending = true; - } - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - return OMPI_ERR_OUT_OF_RESOURCE; - } - } else { - num_fail = 0; - prev_bytes_remaining = bytes_remaining; - } - - do { - rdma_idx = recvreq->req_rdma_idx; - bml_btl = recvreq->req_rdma[rdma_idx].bml_btl; - reg = recvreq->req_rdma[rdma_idx].btl_reg; - size = recvreq->req_rdma[rdma_idx].length; - if(++recvreq->req_rdma_idx >= recvreq->req_rdma_cnt) - recvreq->req_rdma_idx = 0; - } while(!size); - btl = bml_btl->btl; - - /* makes sure that we don't exceed BTL max rdma size - * if memory is not pinned already */ - if( (NULL == reg) && (btl->btl_rdma_pipeline_frag_size != 0) && - (size > btl->btl_rdma_pipeline_frag_size)) { - size = btl->btl_rdma_pipeline_frag_size; - } - - /* take lock to protect converter against concurrent access - * from unpack */ - OPAL_THREAD_LOCK(&recvreq->lock); - opal_convertor_set_position( &recvreq->req_recv.req_base.req_convertor, - &recvreq->req_rdma_offset ); - - /* prepare a descriptor for RDMA */ - mca_bml_base_prepare_dst(bml_btl, reg, - &recvreq->req_recv.req_base.req_convertor, - MCA_BTL_NO_ORDER, 0, &size, MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | - MCA_BTL_DES_FLAGS_PUT, &dst); - OPAL_THREAD_UNLOCK(&recvreq->lock); - - if(OPAL_UNLIKELY(dst == NULL)) { - continue; - } - - dst->des_cbfunc = mca_pml_bfo_put_completion; - dst->des_cbdata = recvreq; - - seg_size = btl->btl_seg_size * dst->des_local_count; - - /* prepare a descriptor for rdma control message */ - mca_bml_base_alloc(bml_btl, &ctl, MCA_BTL_NO_ORDER, sizeof(mca_pml_bfo_rdma_hdr_t) + seg_size, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK); - - if( OPAL_UNLIKELY(NULL == ctl) ) { - mca_bml_base_free(bml_btl,dst); - continue; - } - ctl->des_cbfunc = mca_pml_bfo_recv_ctl_completion; -#if PML_BFO - ctl->des_cbdata = recvreq; -#endif /* PML_BFO */ - - /* fill in rdma header */ - hdr = (mca_pml_bfo_rdma_hdr_t*)ctl->des_local->seg_addr.pval; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_PUT; - hdr->hdr_common.hdr_flags = - (!recvreq->req_ack_sent) ? MCA_PML_BFO_HDR_TYPE_ACK : 0; - hdr->hdr_req = recvreq->remote_req_send; -#if PML_BFO - hdr->hdr_dst_req.pval = recvreq; /* only needed in the first put message */ -#endif /* PML_BFO */ - hdr->hdr_des.pval = dst; - hdr->hdr_rdma_offset = recvreq->req_rdma_offset; - hdr->hdr_seg_cnt = dst->des_local_count; - - /* copy segments */ - memmove (hdr + 1, dst->des_local, seg_size); - - if(!recvreq->req_ack_sent) - recvreq->req_ack_sent = true; - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_PUT, recvreq->req_recv.req_base.req_proc); - - PERUSE_TRACE_COMM_OMPI_EVENT( PERUSE_COMM_REQ_XFER_CONTINUE, - &(recvreq->req_recv.req_base), size, - PERUSE_RECV); - - /* send rdma request to peer */ - rc = mca_bml_base_send(bml_btl, ctl, MCA_PML_BFO_HDR_TYPE_PUT); - if( OPAL_LIKELY( rc >= 0 ) ) { -#if PML_BFO - if ((btl->btl_flags & MCA_BTL_FLAGS_FAILOVER_SUPPORT) && - (ctl->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) { - recvreq->req_events++; - } -#endif /* PML_BFO */ - /* update request state */ - recvreq->req_rdma_offset += size; - OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_pipeline_depth, 1); - recvreq->req_rdma[rdma_idx].length -= size; - bytes_remaining -= size; - } else { - mca_bml_base_free(bml_btl,ctl); - mca_bml_base_free(bml_btl,dst); - } - } - - return OMPI_SUCCESS; -} - -#define IS_PROB_REQ(R) \ - ((MCA_PML_REQUEST_IPROBE == (R)->req_recv.req_base.req_type) || \ - (MCA_PML_REQUEST_PROBE == (R)->req_recv.req_base.req_type) || \ - (MCA_PML_REQUEST_IMPROBE == (R)->req_recv.req_base.req_type) || \ - (MCA_PML_REQUEST_MPROBE == (R)->req_recv.req_base.req_type)) -#define IS_MPROB_REQ(R) \ - ((MCA_PML_REQUEST_IMPROBE == (R)->req_recv.req_base.req_type) || \ - (MCA_PML_REQUEST_MPROBE == (R)->req_recv.req_base.req_type)) - -static inline void append_recv_req_to_queue(opal_list_t *queue, - mca_pml_bfo_recv_request_t *req) -{ - if(OPAL_UNLIKELY(req->req_recv.req_base.req_type == MCA_PML_REQUEST_IPROBE || - req->req_recv.req_base.req_type == MCA_PML_REQUEST_IMPROBE)) - return; - - opal_list_append(queue, (opal_list_item_t*)req); - - /** - * We don't want to generate this kind of event for MPI_Probe. Hopefully, - * the compiler will optimize out the empty if loop in the case where PERUSE - * support is not required by the user. - */ - if(req->req_recv.req_base.req_type != MCA_PML_REQUEST_PROBE || - req->req_recv.req_base.req_type != MCA_PML_REQUEST_MPROBE) { - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_REQ_INSERT_IN_POSTED_Q, - &(req->req_recv.req_base), PERUSE_RECV); - } -} - -/* - * this routine tries to match a posted receive. If a match is found, - * it places the request in the appropriate matched receive list. This - * function has to be called with the communicator matching lock held. -*/ -static mca_pml_bfo_recv_frag_t* -recv_req_match_specific_proc( const mca_pml_bfo_recv_request_t *req, - mca_pml_bfo_comm_proc_t *proc ) -{ - opal_list_t* unexpected_frags = &proc->unexpected_frags; - opal_list_item_t *i; - mca_pml_bfo_recv_frag_t* frag; - int tag = req->req_recv.req_base.req_tag; - - if(opal_list_get_size(unexpected_frags) == 0) - return NULL; - - if( OMPI_ANY_TAG == tag ) { - for (i = opal_list_get_first(unexpected_frags); - i != opal_list_get_end(unexpected_frags); - i = opal_list_get_next(i)) { - frag = (mca_pml_bfo_recv_frag_t*)i; - - if( frag->hdr.hdr_match.hdr_tag >= 0 ) - return frag; - } - } else { - for (i = opal_list_get_first(unexpected_frags); - i != opal_list_get_end(unexpected_frags); - i = opal_list_get_next(i)) { - frag = (mca_pml_bfo_recv_frag_t*)i; - - if( frag->hdr.hdr_match.hdr_tag == tag ) - return frag; - } - } - return NULL; -} - -/* - * this routine is used to try and match a wild posted receive - where - * wild is determined by the value assigned to the source process -*/ -static mca_pml_bfo_recv_frag_t* -recv_req_match_wild( mca_pml_bfo_recv_request_t* req, - mca_pml_bfo_comm_proc_t **p) -{ - mca_pml_bfo_comm_t* comm = req->req_recv.req_base.req_comm->c_pml_comm; - mca_pml_bfo_comm_proc_t* proc = comm->procs; - size_t i; - - /* - * Loop over all the outstanding messages to find one that matches. - * There is an outer loop over lists of messages from each - * process, then an inner loop over the messages from the - * process. - * - * In order to avoid starvation do this in a round-robin fashion. - */ - for (i = comm->last_probed + 1; i < comm->num_procs; i++) { - mca_pml_bfo_recv_frag_t* frag; - - /* loop over messages from the current proc */ - if((frag = recv_req_match_specific_proc(req, &proc[i]))) { - *p = &proc[i]; - comm->last_probed = i; - req->req_recv.req_base.req_proc = proc[i].ompi_proc; - prepare_recv_req_converter(req); - return frag; /* match found */ - } - } - for (i = 0; i <= comm->last_probed; i++) { - mca_pml_bfo_recv_frag_t* frag; - - /* loop over messages from the current proc */ - if((frag = recv_req_match_specific_proc(req, &proc[i]))) { - *p = &proc[i]; - comm->last_probed = i; - req->req_recv.req_base.req_proc = proc[i].ompi_proc; - prepare_recv_req_converter(req); - return frag; /* match found */ - } - } - - *p = NULL; - return NULL; -} - - -void mca_pml_bfo_recv_req_start(mca_pml_bfo_recv_request_t *req) -{ - mca_pml_bfo_comm_t* comm = req->req_recv.req_base.req_comm->c_pml_comm; - mca_pml_bfo_comm_proc_t* proc; - mca_pml_bfo_recv_frag_t* frag; - opal_list_t *queue; - mca_pml_bfo_hdr_t* hdr; - - /* init/re-init the request */ - req->req_lock = 0; - req->req_pipeline_depth = 0; - req->req_bytes_received = 0; - req->req_bytes_expected = 0; - /* What about req_rdma_cnt ? */ -#if PML_BFO - req->req_rdma_cnt = 0; - req->req_events = 0; - req->req_restartseq = 0; - req->req_errstate = 0; -#endif /* PML_BFO */ - req->req_rdma_idx = 0; - req->req_pending = false; - req->req_ack_sent = false; - - MCA_PML_BASE_RECV_START(&req->req_recv.req_base); - - OPAL_THREAD_LOCK(&comm->matching_lock); - /** - * The laps of time between the ACTIVATE event and the SEARCH_UNEX one include - * the cost of the request lock. - */ - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_SEARCH_UNEX_Q_BEGIN, - &(req->req_recv.req_base), PERUSE_RECV); - - /* assign sequence number */ - req->req_recv.req_base.req_sequence = comm->recv_sequence++; - - /* attempt to match posted recv */ - if(req->req_recv.req_base.req_peer == OMPI_ANY_SOURCE) { - frag = recv_req_match_wild(req, &proc); - queue = &comm->wild_receives; -#if !OPAL_ENABLE_HETEROGENEOUS_SUPPORT - /* As we are in a homogeneous environment we know that all remote - * architectures are exactly the same as the local one. Therefore, - * we can safely construct the convertor based on the proc - * information of rank 0. - */ - if( NULL == frag ) { - req->req_recv.req_base.req_proc = ompi_proc_local_proc; - prepare_recv_req_converter(req); - } -#endif /* !OPAL_ENABLE_HETEROGENEOUS_SUPPORT */ - } else { - proc = &comm->procs[req->req_recv.req_base.req_peer]; - req->req_recv.req_base.req_proc = proc->ompi_proc; - frag = recv_req_match_specific_proc(req, proc); - queue = &proc->specific_receives; - /* wild cardrecv will be prepared on match */ - prepare_recv_req_converter(req); - } - - if(OPAL_UNLIKELY(NULL == frag)) { - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_SEARCH_UNEX_Q_END, - &(req->req_recv.req_base), PERUSE_RECV); - /* We didn't find any matches. Record this irecv so we can match - it when the message comes in. */ - append_recv_req_to_queue(queue, req); - req->req_match_received = false; - OPAL_THREAD_UNLOCK(&comm->matching_lock); - } else { - if(OPAL_LIKELY(!IS_PROB_REQ(req))) { - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_REQ_MATCH_UNEX, - &(req->req_recv.req_base), PERUSE_RECV); - - hdr = (mca_pml_bfo_hdr_t*)frag->segments->seg_addr.pval; - PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_REMOVE_FROM_UNEX_Q, - req->req_recv.req_base.req_comm, - hdr->hdr_match.hdr_src, - hdr->hdr_match.hdr_tag, - PERUSE_RECV); - - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_SEARCH_UNEX_Q_END, - &(req->req_recv.req_base), PERUSE_RECV); - - opal_list_remove_item(&proc->unexpected_frags, - (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&comm->matching_lock); - - switch(hdr->hdr_common.hdr_type) { - case MCA_PML_BFO_HDR_TYPE_MATCH: - mca_pml_bfo_recv_request_progress_match(req, frag->btl, frag->segments, - frag->num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RNDV: - mca_pml_bfo_recv_request_progress_rndv(req, frag->btl, frag->segments, - frag->num_segments); - break; - case MCA_PML_BFO_HDR_TYPE_RGET: - mca_pml_bfo_recv_request_progress_rget(req, frag->btl, frag->segments, - frag->num_segments); - break; - default: - assert(0); - } - - MCA_PML_BFO_RECV_FRAG_RETURN(frag); - - } else if (OPAL_UNLIKELY(IS_MPROB_REQ(req))) { - /* Remove the fragment from the match list, as it's now - matched. Stash it somewhere in the request (which, - yes, is a complete hack), where it will be plucked out - during the end of mprobe. The request will then be - "recreated" as a receive request, and the frag will be - restarted with this request during mrecv */ - opal_list_remove_item(&proc->unexpected_frags, - (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&comm->matching_lock); - - req->req_recv.req_base.req_addr = frag; - mca_pml_bfo_recv_request_matched_probe(req, frag->btl, - frag->segments, frag->num_segments); - - } else { - OPAL_THREAD_UNLOCK(&comm->matching_lock); - mca_pml_bfo_recv_request_matched_probe(req, frag->btl, - frag->segments, frag->num_segments); - } - } -} diff --git a/ompi/mca/pml/bfo/pml_bfo_recvreq.h b/ompi/mca/pml/bfo/pml_bfo_recvreq.h deleted file mode 100644 index 7b3a6db627..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_recvreq.h +++ /dev/null @@ -1,449 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011-2012 Los Alamos National Security, LLC. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ -#ifndef OMPI_PML_BFO_RECV_REQUEST_H -#define OMPI_PML_BFO_RECV_REQUEST_H - -#include "pml_bfo.h" -#include "pml_bfo_rdma.h" -#include "pml_bfo_rdmafrag.h" -#include "ompi/proc/proc.h" -#include "ompi/mca/pml/bfo/pml_bfo_comm.h" -#include "opal/mca/mpool/base/base.h" -#include "ompi/mca/pml/base/pml_base_recvreq.h" -#if PML_BFO -#define RECVREQ_RECVERRSENT 0x01 -#define RECVREQ_RNDVRESTART_RECVED 0x02 -#define RECVREQ_RNDVRESTART_ACKED 0x04 -#endif /* PML_BFO */ - -BEGIN_C_DECLS - -struct mca_pml_bfo_recv_request_t { - mca_pml_base_recv_request_t req_recv; - opal_ptr_t remote_req_send; -#if PML_BFO - int32_t req_msgseq; /* PML sequence number */ - int32_t req_events; /* number of outstanding events on request */ - int32_t req_restartseq; /* sequence number of restarted request */ - int32_t req_errstate; /* state of request if in error */ -#endif /* PML_BFO */ - int32_t req_lock; - size_t req_pipeline_depth; - size_t req_bytes_received; /**< amount of data transferred into the user buffer */ - size_t req_bytes_expected; /**< local size of the data as suggested by the user */ - size_t req_rdma_offset; - size_t req_send_offset; - uint32_t req_rdma_cnt; - uint32_t req_rdma_idx; - bool req_pending; - bool req_ack_sent; /**< whether ack was sent to the sender */ - bool req_match_received; /**< Prevent request to be completed prematurely */ - opal_mutex_t lock; - mca_pml_bfo_com_btl_t req_rdma[1]; -}; -typedef struct mca_pml_bfo_recv_request_t mca_pml_bfo_recv_request_t; - -OBJ_CLASS_DECLARATION(mca_pml_bfo_recv_request_t); - -static inline bool lock_recv_request(mca_pml_bfo_recv_request_t *recvreq) -{ - return OPAL_THREAD_ADD_FETCH32(&recvreq->req_lock, 1) == 1; -} - -static inline bool unlock_recv_request(mca_pml_bfo_recv_request_t *recvreq) -{ - return OPAL_THREAD_ADD_FETCH32(&recvreq->req_lock, -1) == 0; -} - -/** - * Allocate a recv request from the modules free list. - * - * @param rc (OUT) OMPI_SUCCESS or error status on failure. - * @return Receive request. - */ -#define MCA_PML_BFO_RECV_REQUEST_ALLOC(recvreq) \ -do { \ - ompi_free_list_item_t* item; \ - OMPI_FREE_LIST_GET_MT(&mca_pml_base_recv_requests, item); \ - recvreq = (mca_pml_bfo_recv_request_t*)item; \ -} while(0) - - -/** - * Initialize a receive request with call parameters. - * - * @param request (IN) Receive request. - * @param addr (IN) User buffer. - * @param count (IN) Number of elements of indicated datatype. - * @param datatype (IN) User defined datatype. - * @param src (IN) Source rank w/in the communicator. - * @param tag (IN) User defined tag. - * @param comm (IN) Communicator. - * @param persistent (IN) Is this a ersistent request. - */ -#define MCA_PML_BFO_RECV_REQUEST_INIT( request, \ - addr, \ - count, \ - datatype, \ - src, \ - tag, \ - comm, \ - persistent) \ -do { \ - MCA_PML_BASE_RECV_REQUEST_INIT( &(request)->req_recv, \ - addr, \ - count, \ - datatype, \ - src, \ - tag, \ - comm, \ - persistent); \ -} while(0) - -/** - * Mark the request as completed at MPI level for internal purposes. - * - * @param recvreq (IN) Receive request. - */ -#define MCA_PML_BFO_RECV_REQUEST_MPI_COMPLETE( recvreq ) \ - do { \ - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE, \ - &(recvreq->req_recv.req_base), PERUSE_RECV ); \ - ompi_request_complete( &(recvreq->req_recv.req_base.req_ompi), true ); \ - } while (0) - -/* - * Free the PML receive request - */ -#define MCA_PML_BFO_RECV_REQUEST_RETURN(recvreq) \ - { \ - MCA_PML_BASE_RECV_REQUEST_FINI(&(recvreq)->req_recv); \ - OPAL_FREE_LIST_RETURN( &mca_pml_base_recv_requests, \ - (opal_free_list_item_t*)(recvreq)); \ - } - -/** - * Complete receive request. Request structure cannot be accessed after calling - * this function any more. - * - * @param recvreq (IN) Receive request. - */ -static inline void -recv_request_pml_complete(mca_pml_bfo_recv_request_t *recvreq) -{ - size_t i; - - if(false == recvreq->req_recv.req_base.req_pml_complete) { - - if(recvreq->req_recv.req_bytes_packed > 0) { - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END, - &recvreq->req_recv.req_base, PERUSE_RECV ); - } - - for(i = 0; i < recvreq->req_rdma_cnt; i++) { - mca_mpool_base_registration_t* btl_reg = recvreq->req_rdma[i].btl_reg; - if( NULL != btl_reg && btl_reg->mpool != NULL) { - btl_reg->mpool->mpool_deregister( btl_reg->mpool, btl_reg ); - } - } - recvreq->req_rdma_cnt = 0; -#if PML_BFO - recvreq->req_msgseq -= 100; -#endif /* PML_BFO */ - - if(true == recvreq->req_recv.req_base.req_free_called) { - if( MPI_SUCCESS != recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR ) { - ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST); - } - MCA_PML_BFO_RECV_REQUEST_RETURN(recvreq); - } else { - /* initialize request status */ - recvreq->req_recv.req_base.req_pml_complete = true; - recvreq->req_recv.req_base.req_ompi.req_status._ucount = - recvreq->req_bytes_received; - if (recvreq->req_recv.req_bytes_packed > recvreq->req_bytes_expected) { - recvreq->req_recv.req_base.req_ompi.req_status._ucount = - recvreq->req_recv.req_bytes_packed; - recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR = - MPI_ERR_TRUNCATE; - } - MCA_PML_BFO_RECV_REQUEST_MPI_COMPLETE(recvreq); - } - } -} - -static inline bool -recv_request_pml_complete_check(mca_pml_bfo_recv_request_t *recvreq) -{ -#if OPAL_ENABLE_MULTI_THREADS - opal_atomic_rmb(); -#endif - if(recvreq->req_match_received && - recvreq->req_bytes_received >= recvreq->req_recv.req_bytes_packed && -#if PML_BFO - (0 == recvreq->req_events) && lock_recv_request(recvreq)) { -#else /* PML_BFO */ - lock_recv_request(recvreq)) { -#endif /* PML_BFO */ - recv_request_pml_complete(recvreq); - return true; - } - - return false; -} - -extern void mca_pml_bfo_recv_req_start(mca_pml_bfo_recv_request_t *req); -#define MCA_PML_BFO_RECV_REQUEST_START(r) mca_pml_bfo_recv_req_start(r) - -static inline void prepare_recv_req_converter(mca_pml_bfo_recv_request_t *req) -{ - if( req->req_recv.req_base.req_datatype->super.size | req->req_recv.req_base.req_count ) { - opal_convertor_copy_and_prepare_for_recv( - req->req_recv.req_base.req_proc->super.proc_convertor, - &(req->req_recv.req_base.req_datatype->super), - req->req_recv.req_base.req_count, - req->req_recv.req_base.req_addr, - 0, - &req->req_recv.req_base.req_convertor); - opal_convertor_get_unpacked_size(&req->req_recv.req_base.req_convertor, - &req->req_bytes_expected); - } -} - -#define MCA_PML_BFO_RECV_REQUEST_MATCHED(request, hdr) \ - recv_req_matched(request, hdr) - -static inline void recv_req_matched(mca_pml_bfo_recv_request_t *req, - mca_pml_bfo_match_hdr_t *hdr) -{ - req->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src; - req->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag; - req->req_match_received = true; -#if PML_BFO - req->req_msgseq = hdr->hdr_seq; -#endif /* PML_BFO */ -#if OPAL_ENABLE_MULTI_THREADS - opal_atomic_wmb(); -#endif - if(req->req_recv.req_bytes_packed > 0) { -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - if(MPI_ANY_SOURCE == req->req_recv.req_base.req_peer) { - /* non wildcard prepared during post recv */ - prepare_recv_req_converter(req); - } -#endif /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT */ - PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_REQ_XFER_BEGIN, - &req->req_recv.req_base, PERUSE_RECV); - } -} - - -/** - * - */ - -#define MCA_PML_BFO_RECV_REQUEST_UNPACK( request, \ - segments, \ - num_segments, \ - seg_offset, \ - data_offset, \ - bytes_received, \ - bytes_delivered) \ -do { \ - bytes_delivered = 0; \ - if(request->req_recv.req_bytes_packed > 0) { \ - struct iovec iov[MCA_BTL_DES_MAX_SEGMENTS]; \ - uint32_t iov_count = 0; \ - size_t max_data = bytes_received; \ - size_t n, offset = seg_offset; \ - mca_btl_base_segment_t* segment = segments; \ - \ - OPAL_THREAD_LOCK(&request->lock); \ - for( n = 0; n < num_segments; n++, segment++ ) { \ - if(offset >= segment->seg_len) { \ - offset -= segment->seg_len; \ - } else { \ - iov[iov_count].iov_len = segment->seg_len - offset; \ - iov[iov_count].iov_base = (IOVBASE_TYPE*) \ - ((unsigned char*)segment->seg_addr.pval + offset); \ - iov_count++; \ - offset = 0; \ - } \ - } \ - PERUSE_TRACE_COMM_OMPI_EVENT (PERUSE_COMM_REQ_XFER_CONTINUE, \ - &(recvreq->req_recv.req_base), max_data, \ - PERUSE_RECV); \ - opal_convertor_set_position( &(request->req_recv.req_base.req_convertor), \ - &data_offset ); \ - opal_convertor_unpack( &(request)->req_recv.req_base.req_convertor, \ - iov, \ - &iov_count, \ - &max_data ); \ - bytes_delivered = max_data; \ - OPAL_THREAD_UNLOCK(&request->lock); \ - } \ -} while (0) - - -/** - * - */ - -void mca_pml_bfo_recv_request_progress_match( - mca_pml_bfo_recv_request_t* req, - struct mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments); - -/** - * - */ - -void mca_pml_bfo_recv_request_progress_frag( - mca_pml_bfo_recv_request_t* req, - struct mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments); - -/** - * - */ - -void mca_pml_bfo_recv_request_progress_rndv( - mca_pml_bfo_recv_request_t* req, - struct mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments); - -/** - * - */ - -void mca_pml_bfo_recv_request_progress_rget( - mca_pml_bfo_recv_request_t* req, - struct mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments); - -/** - * - */ - -void mca_pml_bfo_recv_request_matched_probe( - mca_pml_bfo_recv_request_t* req, - struct mca_btl_base_module_t* btl, - mca_btl_base_segment_t* segments, - size_t num_segments); - -/** - * - */ - -int mca_pml_bfo_recv_request_schedule_once( - mca_pml_bfo_recv_request_t* req, mca_bml_base_btl_t* start_bml_btl); - -static inline int mca_pml_bfo_recv_request_schedule_exclusive( - mca_pml_bfo_recv_request_t* req, - mca_bml_base_btl_t* start_bml_btl) -{ - int rc; - - do { - rc = mca_pml_bfo_recv_request_schedule_once(req, start_bml_btl); - if(rc == OMPI_ERR_OUT_OF_RESOURCE) - break; - } while(!unlock_recv_request(req)); - - if(OMPI_SUCCESS == rc) - recv_request_pml_complete_check(req); - - return rc; -} - -static inline void mca_pml_bfo_recv_request_schedule( - mca_pml_bfo_recv_request_t* req, - mca_bml_base_btl_t* start_bml_btl) -{ - if(!lock_recv_request(req)) - return; - - (void)mca_pml_bfo_recv_request_schedule_exclusive(req, start_bml_btl); -} - -#define MCA_PML_BFO_ADD_ACK_TO_PENDING(P, S, D, O) \ - do { \ - mca_pml_bfo_pckt_pending_t *_pckt; \ - \ - MCA_PML_BFO_PCKT_PENDING_ALLOC(_pckt); \ - _pckt->hdr.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_ACK; \ - _pckt->hdr.hdr_ack.hdr_src_req.lval = (S); \ - _pckt->hdr.hdr_ack.hdr_dst_req.pval = (D); \ - _pckt->hdr.hdr_ack.hdr_send_offset = (O); \ - _pckt->proc = (P); \ - _pckt->bml_btl = NULL; \ - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); \ - opal_list_append(&mca_pml_bfo.pckt_pending, \ - (opal_list_item_t*)_pckt); \ - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); \ - } while(0) - -int mca_pml_bfo_recv_request_ack_send_btl(ompi_proc_t* proc, - mca_bml_base_btl_t* bml_btl, uint64_t hdr_src_req, void *hdr_dst_req, - uint64_t hdr_rdma_offset, bool nordma); - -static inline int mca_pml_bfo_recv_request_ack_send(ompi_proc_t* proc, - uint64_t hdr_src_req, void *hdr_dst_req, uint64_t hdr_send_offset, - bool nordma) -{ - size_t i; - mca_bml_base_btl_t* bml_btl; - mca_bml_base_endpoint_t* endpoint = - (mca_bml_base_endpoint_t*)proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - - for(i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) { - bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager); - if(mca_pml_bfo_recv_request_ack_send_btl(proc, bml_btl, hdr_src_req, - hdr_dst_req, hdr_send_offset, nordma) == OMPI_SUCCESS) - return OMPI_SUCCESS; - } - - MCA_PML_BFO_ADD_ACK_TO_PENDING(proc, hdr_src_req, hdr_dst_req, - hdr_send_offset); - - return OMPI_ERR_OUT_OF_RESOURCE; -} - -int mca_pml_bfo_recv_request_get_frag(mca_pml_bfo_rdma_frag_t* frag); - -/* This function tries to continue recvreq that stuck due to resource - * unavailability. Recvreq is added to recv_pending list if scheduling of put - * operation cannot be accomplished for some reason. */ -void mca_pml_bfo_recv_request_process_pending(void); - -END_C_DECLS - -#endif - diff --git a/ompi/mca/pml/bfo/pml_bfo_sendreq.c b/ompi/mca/pml/bfo/pml_bfo_sendreq.c deleted file mode 100644 index 176eadf4f6..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_sendreq.c +++ /dev/null @@ -1,1401 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011-2012 Los Alamos National Security, LLC. - * All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "opal/prefetch.h" -#include "opal/mca/mpool/mpool.h" -#include "ompi/constants.h" -#include "ompi/mca/pml/pml.h" -#include "pml_bfo.h" -#include "pml_bfo_hdr.h" -#include "pml_bfo_sendreq.h" -#include "pml_bfo_rdmafrag.h" -#include "pml_bfo_recvreq.h" -#if PML_BFO -#include "pml_bfo_failover.h" -#endif /* PML_BFO */ -#include "ompi/mca/bml/base/base.h" -#include "ompi/memchecker.h" - -OBJ_CLASS_INSTANCE(mca_pml_bfo_send_range_t, ompi_free_list_item_t, - NULL, NULL); - -void mca_pml_bfo_send_request_process_pending(mca_bml_base_btl_t *bml_btl) -{ - int rc, i, s = opal_list_get_size(&mca_pml_bfo.send_pending); - - /* advance pending requests */ - for(i = 0; i < s; i++) { - mca_pml_bfo_send_pending_t pending_type = MCA_PML_BFO_SEND_PENDING_NONE; - mca_pml_bfo_send_request_t* sendreq; - mca_bml_base_btl_t *send_dst; - - sendreq = get_request_from_send_pending(&pending_type); - if(OPAL_UNLIKELY(NULL == sendreq)) - break; - - switch(pending_type) { - case MCA_PML_BFO_SEND_PENDING_SCHEDULE: - rc = mca_pml_bfo_send_request_schedule_exclusive(sendreq); - if(OMPI_ERR_OUT_OF_RESOURCE == rc) { - return; - } - break; - case MCA_PML_BFO_SEND_PENDING_START: - send_dst = mca_bml_base_btl_array_find( - &sendreq->req_endpoint->btl_eager, bml_btl->btl); - if (NULL == send_dst) { - /* Put request back onto pending list and try next one. */ - add_request_to_send_pending(sendreq, - MCA_PML_BFO_SEND_PENDING_START, true); - } else { - rc = mca_pml_bfo_send_request_start_btl(sendreq, send_dst); - if (OMPI_ERR_OUT_OF_RESOURCE == rc) { - /* No more resources on this btl so prepend to the pending - * list to minimize reordering and give up for now. */ - add_request_to_send_pending(sendreq, - MCA_PML_BFO_SEND_PENDING_START, false); - return; - } - } - break; - default: - opal_output(0, "[%s:%d] wrong send request type\n", - __FILE__, __LINE__); - break; - } - } -} - -/* - * The free call mark the final stage in a request life-cycle. Starting from this - * point the request is completed at both PML and user level, and can be used - * for others p2p communications. Therefore, in the case of the BFO PML it should - * be added to the free request list. - */ -static int mca_pml_bfo_send_request_free(struct ompi_request_t** request) -{ - mca_pml_bfo_send_request_t* sendreq = *(mca_pml_bfo_send_request_t**)request; - if( false == sendreq->req_send.req_base.req_free_called ) { - - sendreq->req_send.req_base.req_free_called = true; - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_NOTIFY, - &(sendreq->req_send.req_base), PERUSE_SEND ); - - if( true == sendreq->req_send.req_base.req_pml_complete ) { - /* make buffer defined when the request is compeleted, - and before releasing the objects. */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - - MCA_PML_BFO_SEND_REQUEST_RETURN( sendreq ); - } - *request = MPI_REQUEST_NULL; - } - return OMPI_SUCCESS; -} - -static int mca_pml_bfo_send_request_cancel(struct ompi_request_t* request, int complete) -{ - /* we dont cancel send requests by now */ - return OMPI_SUCCESS; -} - -static void mca_pml_bfo_send_request_construct(mca_pml_bfo_send_request_t* req) -{ - req->req_send.req_base.req_type = MCA_PML_REQUEST_SEND; - req->req_send.req_base.req_ompi.req_start = mca_pml_bfo_start; - req->req_send.req_base.req_ompi.req_free = mca_pml_bfo_send_request_free; - req->req_send.req_base.req_ompi.req_cancel = mca_pml_bfo_send_request_cancel; - req->req_rdma_cnt = 0; - req->req_throttle_sends = false; - OBJ_CONSTRUCT(&req->req_send_ranges, opal_list_t); - OBJ_CONSTRUCT(&req->req_send_range_lock, opal_mutex_t); -} - -static void mca_pml_bfo_send_request_destruct(mca_pml_bfo_send_request_t* req) -{ - OBJ_DESTRUCT(&req->req_send_ranges); - OBJ_DESTRUCT(&req->req_send_range_lock); -} - -OBJ_CLASS_INSTANCE( mca_pml_bfo_send_request_t, - mca_pml_base_send_request_t, - mca_pml_bfo_send_request_construct, - mca_pml_bfo_send_request_destruct ); - -/** - * Completion of a short message - nothing left to schedule. - */ - -static inline void -mca_pml_bfo_match_completion_free_request( mca_bml_base_btl_t* bml_btl, - mca_pml_bfo_send_request_t* sendreq ) -{ - if( sendreq->req_send.req_bytes_packed > 0 ) { - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_BEGIN, - &(sendreq->req_send.req_base), PERUSE_SEND ); - } - - /* signal request completion */ - send_request_pml_complete(sendreq); - - /* check for pending requests */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -static void -mca_pml_bfo_match_completion_free( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)des->des_cbdata; - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context; - - /* check completion status */ - if( OPAL_UNLIKELY(OMPI_SUCCESS != status) ) { -#if PML_BFO - mca_pml_bfo_repost_match_fragment(des); - return; -#else /* PML_BFO */ - /* TSW - FIX */ - opal_output(0, "%s:%d FATAL", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif /* PML_BFO */ - } -#if PML_BFO - MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, "MATCH"); -#endif /* PML_BFO */ - mca_pml_bfo_match_completion_free_request( bml_btl, sendreq ); -} - -static inline void -mca_pml_bfo_rndv_completion_request( mca_bml_base_btl_t* bml_btl, - mca_pml_bfo_send_request_t* sendreq, - size_t req_bytes_delivered ) -{ - if( sendreq->req_send.req_bytes_packed > 0 ) { - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_BEGIN, - &(sendreq->req_send.req_base), PERUSE_SEND ); - } - - OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, req_bytes_delivered); - - /* advance the request */ - OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, -1); - - send_request_pml_complete_check(sendreq); - - /* check for pending requests */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -/* - * Completion of the first fragment of a long message that - * requires an acknowledgement - */ -static void -mca_pml_bfo_rndv_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)des->des_cbdata; - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*)des->des_context; - size_t req_bytes_delivered; - - /* check completion status */ - if( OPAL_UNLIKELY(OMPI_SUCCESS != status) ) { -#if PML_BFO - if (true == mca_pml_bfo_rndv_completion_status_error(des, sendreq)) - return; -#else /* PML_BFO */ - /* TSW - FIX */ - opal_output(0, "%s:%d FATAL", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif /* PML_BFO */ - } -#if PML_BFO - sendreq->req_events--; - MCA_PML_BFO_RNDV_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl, - MCA_PML_BFO_HDR_TYPE_RNDV, "RNDV"); -#endif /* PML_BFO */ - - /* count bytes of user data actually delivered. As the rndv completion only - * happens in one thread, the increase of the req_bytes_delivered does not - * have to be atomic. - */ - req_bytes_delivered = mca_pml_bfo_compute_segment_length (btl->btl_seg_size, - (void *) des->des_local, - des->des_local_count, - sizeof(mca_pml_bfo_rendezvous_hdr_t)); - -#if PML_BFO - MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, "RNDV"); -#endif /* PML_BFO */ - mca_pml_bfo_rndv_completion_request( bml_btl, sendreq, req_bytes_delivered ); -} - - -/** - * Completion of a get request. - */ - -static void -mca_pml_bfo_rget_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)des->des_cbdata; - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*)des->des_context; - size_t req_bytes_delivered; -#if PML_BFO - MCA_PML_BFO_RGET_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, btl, des); -#endif /* PML_BFO */ - - /* count bytes of user data actually delivered and check for request completion */ - req_bytes_delivered = mca_pml_bfo_compute_segment_length (btl->btl_seg_size, - (void *) des->des_local, - des->des_local_count, 0); - if (OPAL_LIKELY(0 < req_bytes_delivered)) { - OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, req_bytes_delivered); - } - - send_request_pml_complete_check(sendreq); - /* free the descriptor */ -#if PML_BFO - btl->btl_free(btl, des); - MCA_PML_BFO_CHECK_SENDREQ_RDMA_BML_BTL(bml_btl, btl, sendreq, "RGET"); -#else /* PML_BFO */ - mca_bml_base_free(bml_btl, des); -#endif /* PML_BFO */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - - -/** - * Completion of a control message - return resources. - */ - -static void -mca_pml_bfo_send_ctl_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context; - -#if PML_BFO - if(OPAL_UNLIKELY(OMPI_SUCCESS != status)) { - mca_pml_bfo_send_ctl_completion_status_error(des); - return; - } - MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, des->des_cbdata, "RGET"); -#endif /* PML_BFO */ - /* check for pending requests */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -/** - * Completion of additional fragments of a large message - may need - * to schedule additional fragments. - */ - -static void -mca_pml_bfo_frag_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)des->des_cbdata; - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context; - size_t req_bytes_delivered; -#if PML_BFO - sendreq->req_events--; -#endif /* PML_BFO */ - - /* check completion status */ - if( OPAL_UNLIKELY(OMPI_SUCCESS != status) ) { -#if PML_BFO - sendreq->req_error++; -#else /* PML_BFO */ - /* TSW - FIX */ - opal_output(0, "%s:%d FATAL", __FILE__, __LINE__); - ompi_rte_abort(-1, NULL); -#endif /* PML_BFO */ - } - - /* count bytes of user data actually delivered */ - req_bytes_delivered = mca_pml_bfo_compute_segment_length (btl->btl_seg_size, - (void *) des->des_local, - des->des_local_count, - sizeof(mca_pml_bfo_frag_hdr_t)); - - OPAL_THREAD_SUB_FETCH_SIZE_T(&sendreq->req_pipeline_depth, 1); - OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, req_bytes_delivered); - -#if PML_BFO - MCA_PML_BFO_FRAG_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl, - MCA_PML_BFO_HDR_TYPE_FRAG, "FRAG"); -#endif /* PML_BFO */ - if(send_request_pml_complete_check(sendreq) == false) { - mca_pml_bfo_send_request_schedule(sendreq); -#if PML_BFO - MCA_PML_BFO_FRAG_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl, - MCA_PML_BFO_HDR_TYPE_FRAG, - "FRAG (BTL removal)"); -#endif /* PML_BFO */ - } - - /* check for pending requests */ -#if PML_BFO - MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, "FRAG"); -#endif /* PML_BFO */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -/** - * Buffer the entire message and mark as complete. - */ - -int mca_pml_bfo_send_request_start_buffered( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size) -{ - mca_btl_base_descriptor_t* des; - mca_btl_base_segment_t* segment; - mca_pml_bfo_hdr_t* hdr; - struct iovec iov; - unsigned int iov_count; - size_t max_data, req_bytes_delivered; - int rc; - - /* allocate descriptor */ - mca_bml_base_alloc(bml_btl, &des, - MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_rendezvous_hdr_t) + size, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - if( OPAL_UNLIKELY(NULL == des) ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - segment = des->des_local; - - /* pack the data into the BTL supplied buffer */ - iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)segment->seg_addr.pval + - sizeof(mca_pml_bfo_rendezvous_hdr_t)); - iov.iov_len = size; - iov_count = 1; - max_data = size; - if((rc = opal_convertor_pack( &sendreq->req_send.req_base.req_convertor, - &iov, - &iov_count, - &max_data)) < 0) { - mca_bml_base_free(bml_btl, des); - return rc; - } - req_bytes_delivered = max_data; - - /* build rendezvous header */ - hdr = (mca_pml_bfo_hdr_t*)segment->seg_addr.pval; - hdr->hdr_common.hdr_flags = 0; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RNDV; - hdr->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; - hdr->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; - hdr->hdr_rndv.hdr_src_req.pval = sendreq; -#if PML_BFO - MCA_PML_BFO_CHECK_FOR_RNDV_RESTART(hdr, sendreq, "RNDV(buffered)"); -#endif /* PML_BFO */ - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_RNDV, - sendreq->req_send.req_base.req_proc); - - /* update lengths */ - segment->seg_len = sizeof(mca_pml_bfo_rendezvous_hdr_t) + max_data; - - des->des_cbfunc = mca_pml_bfo_rndv_completion; - des->des_cbdata = sendreq; - - /* buffer the remainder of the message */ - rc = mca_pml_base_bsend_request_alloc((ompi_request_t*)sendreq); - if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { - mca_bml_base_free(bml_btl, des); - return rc; - } - - iov.iov_base = (IOVBASE_TYPE*)(((unsigned char*)sendreq->req_send.req_addr) + max_data); - iov.iov_len = max_data = sendreq->req_send.req_bytes_packed - max_data; - - if((rc = opal_convertor_pack( &sendreq->req_send.req_base.req_convertor, - &iov, - &iov_count, - &max_data)) < 0) { - mca_bml_base_free(bml_btl, des); - return rc; - } - - /* re-init convertor for packed data */ - opal_convertor_prepare_for_send( &sendreq->req_send.req_base.req_convertor, - &(ompi_mpi_byte.dt.super), - sendreq->req_send.req_bytes_packed, - sendreq->req_send.req_addr ); - - /* wait for ack and completion */ - sendreq->req_state = 2; - - /* request is complete at mpi level */ - MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq, true); - - /* send */ - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_RNDV); - if( OPAL_LIKELY( rc >= 0 ) ) { - if( OPAL_LIKELY( 1 == rc ) ) { - mca_pml_bfo_rndv_completion_request( bml_btl, sendreq, req_bytes_delivered); - } -#if PML_BFO - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - sendreq->req_events++; - } -#endif /* PML_BFO */ - return OMPI_SUCCESS; - } - mca_bml_base_free(bml_btl, des ); - return rc; -} - - -/** - * We work on a buffered request with a size smaller than the eager size - * or the BTL is not able to send the data IN_PLACE. Request a segment - * that is used for initial hdr and any eager data. This is used only - * from the _START macro. - */ -int mca_pml_bfo_send_request_start_copy( mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size ) -{ - mca_btl_base_descriptor_t* des = NULL; - mca_btl_base_segment_t* segment; - mca_pml_bfo_hdr_t* hdr; - struct iovec iov; - unsigned int iov_count; - size_t max_data = size; - int rc; - - if(NULL != bml_btl->btl->btl_sendi) { - mca_pml_bfo_match_hdr_t match; - match.hdr_common.hdr_flags = 0; - match.hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_MATCH; - match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - match.hdr_tag = sendreq->req_send.req_base.req_tag; - match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - - bfo_hdr_hton(&match, MCA_PML_BFO_HDR_TYPE_MATCH, - sendreq->req_send.req_base.req_proc); - - /* try to send immediately */ - rc = mca_bml_base_sendi( bml_btl, &sendreq->req_send.req_base.req_convertor, - &match, OMPI_PML_BFO_MATCH_HDR_LEN, - size, MCA_BTL_NO_ORDER, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP, - MCA_PML_BFO_HDR_TYPE_MATCH, - &des); - if( OPAL_LIKELY(OMPI_SUCCESS == rc) ) { -#if PML_BFO - /* Needed in case of failover */ - if (NULL != des) { - des->des_cbfunc = mca_pml_bfo_match_completion_free; - des->des_cbdata = sendreq->req_endpoint; - } -#endif /* PML_BFO */ - /* signal request completion */ - send_request_pml_complete(sendreq); - - /* check for pending requests */ - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); - return OMPI_SUCCESS; - } - } else { - /* allocate descriptor */ - mca_bml_base_alloc( bml_btl, &des, - MCA_BTL_NO_ORDER, - OMPI_PML_BFO_MATCH_HDR_LEN + size, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - } - if( OPAL_UNLIKELY(NULL == des) ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - segment = des->des_local; - - if(size > 0) { - /* pack the data into the supplied buffer */ - iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)segment->seg_addr.pval + - OMPI_PML_BFO_MATCH_HDR_LEN); - iov.iov_len = size; - iov_count = 1; - /* - * Before copy the user buffer, make the target part - * accessible. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - (void)opal_convertor_pack( &sendreq->req_send.req_base.req_convertor, - &iov, &iov_count, &max_data ); - /* - * Packing finished, make the user buffer unaccessable. - */ - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - } - - - /* build match header */ - hdr = (mca_pml_bfo_hdr_t*)segment->seg_addr.pval; - hdr->hdr_common.hdr_flags = 0; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_MATCH; - hdr->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; - hdr->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_MATCH, - sendreq->req_send.req_base.req_proc); - - /* update lengths */ - segment->seg_len = OMPI_PML_BFO_MATCH_HDR_LEN + max_data; - - /* short message */ - des->des_cbdata = sendreq; - des->des_cbfunc = mca_pml_bfo_match_completion_free; - - /* send */ - rc = mca_bml_base_send_status(bml_btl, des, MCA_PML_BFO_HDR_TYPE_MATCH); - if( OPAL_LIKELY( rc >= OMPI_SUCCESS ) ) { - if( OPAL_LIKELY( 1 == rc ) ) { - mca_pml_bfo_match_completion_free_request( bml_btl, sendreq ); - } - return OMPI_SUCCESS; - } - if (OMPI_ERR_RESOURCE_BUSY == rc) { - /* No more resources. Allow the upper level to queue the send */ - rc = OMPI_ERR_OUT_OF_RESOURCE; - } - - mca_bml_base_free (bml_btl, des); - - return rc; -} - -/** - * BTL can send directly from user buffer so allow the BTL - * to prepare the segment list. Start sending a small message. - */ - -int mca_pml_bfo_send_request_start_prepare( mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size ) -{ - mca_btl_base_descriptor_t* des; - mca_btl_base_segment_t* segment; - mca_pml_bfo_hdr_t* hdr; - int rc; - - /* prepare descriptor */ - mca_bml_base_prepare_src( bml_btl, - NULL, - &sendreq->req_send.req_base.req_convertor, - MCA_BTL_NO_ORDER, - OMPI_PML_BFO_MATCH_HDR_LEN, - &size, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP, - &des ); - if( OPAL_UNLIKELY(NULL == des) ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - segment = des->des_local; - - /* build match header */ - hdr = (mca_pml_bfo_hdr_t*)segment->seg_addr.pval; - hdr->hdr_common.hdr_flags = 0; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_MATCH; - hdr->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; - hdr->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_MATCH, - sendreq->req_send.req_base.req_proc); - - /* short message */ - des->des_cbfunc = mca_pml_bfo_match_completion_free; - des->des_cbdata = sendreq; - - /* send */ - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_MATCH); - if( OPAL_LIKELY( rc >= 0 ) ) { - if( OPAL_LIKELY( 1 == rc ) ) { - mca_pml_bfo_match_completion_free_request( bml_btl, sendreq ); - } - return OMPI_SUCCESS; - } - mca_bml_base_free(bml_btl, des ); - return rc; -} - - -/** - * We have contigous data that is registered - schedule across - * available nics. - */ - -int mca_pml_bfo_send_request_start_rdma( mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size ) -{ - /* - * When req_rdma array is constructed the first element of the array always - * assigned different btl in round robin fashion (if there are more than - * one RDMA capable BTLs). This way round robin distribution of RDMA - * operation is achieved. - */ - - mca_btl_base_descriptor_t* des, *src = NULL; - mca_btl_base_segment_t* segment; - mca_pml_bfo_hdr_t* hdr; - bool need_local_cb = false; - int rc; - - bml_btl = sendreq->req_rdma[0].bml_btl; - if((sendreq->req_rdma_cnt == 1) && (bml_btl->btl_flags & (MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_CUDA_GET))) { - mca_mpool_base_registration_t* reg = sendreq->req_rdma[0].btl_reg; - size_t seg_size; - size_t old_position = sendreq->req_send.req_base.req_convertor.bConverted; - - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - /* prepare source descriptor/segment(s) */ - /* PML owns this descriptor and will free it in */ - /* get_completion */ - mca_bml_base_prepare_src( bml_btl, - reg, - &sendreq->req_send.req_base.req_convertor, - MCA_BTL_NO_ORDER, - 0, - &size, - MCA_BTL_DES_FLAGS_GET, - &src ); - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - if( OPAL_UNLIKELY(NULL == src) ) { - opal_convertor_set_position(&sendreq->req_send.req_base.req_convertor, - &old_position); - return OMPI_ERR_OUT_OF_RESOURCE; - } - src->des_cbfunc = mca_pml_bfo_rget_completion; - src->des_cbdata = sendreq; - - seg_size = bml_btl->btl->btl_seg_size * src->des_local_count; - - /* allocate space for get hdr + segment list */ - mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_rget_hdr_t) + seg_size, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - if( OPAL_UNLIKELY(NULL == des) ) { - opal_convertor_set_position( &sendreq->req_send.req_base.req_convertor, - &old_position ); - mca_bml_base_free(bml_btl, src); - return OMPI_ERR_OUT_OF_RESOURCE; - } - segment = des->des_local; - - /* build match header */ - hdr = (mca_pml_bfo_hdr_t*)segment->seg_addr.pval; - hdr->hdr_common.hdr_flags = MCA_PML_BFO_HDR_FLAGS_CONTIG|MCA_PML_BFO_HDR_FLAGS_PIN; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RGET; - hdr->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; - hdr->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; - hdr->hdr_rndv.hdr_src_req.pval = sendreq; -#if PML_BFO - MCA_PML_BFO_CHECK_FOR_RNDV_RESTART(hdr, sendreq, "RGET"); -#endif /* PML_BFO */ - hdr->hdr_rget.hdr_des.pval = src; - hdr->hdr_rget.hdr_seg_cnt = src->des_local_count; - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_RGET, - sendreq->req_send.req_base.req_proc); - - /* copy segment data */ - memmove (&hdr->hdr_rget + 1, src->des_local, seg_size); - - des->des_cbfunc = mca_pml_bfo_send_ctl_completion; - - /** - * Well, it's a get so we will not know when the peer get the data anyway. - * If we generate the PERUSE event here, at least we will know when do we - * sent the GET message ... - */ - if( sendreq->req_send.req_bytes_packed > 0 ) { - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_BEGIN, - &(sendreq->req_send.req_base), PERUSE_SEND ); - } - - } else { - - /* allocate a rendezvous header - dont eager send any data - * receiver will schedule rdma put(s) of the entire message - */ - - mca_bml_base_alloc(bml_btl, &des, - MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_rendezvous_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - if( OPAL_UNLIKELY(NULL == des)) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - segment = des->des_local; - - /* build hdr */ - hdr = (mca_pml_bfo_hdr_t*)segment->seg_addr.pval; - hdr->hdr_common.hdr_flags = MCA_PML_BFO_HDR_FLAGS_CONTIG|MCA_PML_BFO_HDR_FLAGS_PIN; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RNDV; - hdr->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; - hdr->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; - hdr->hdr_rndv.hdr_src_req.pval = sendreq; -#if PML_BFO - MCA_PML_BFO_CHECK_FOR_RNDV_RESTART(hdr, sendreq, "RNDV"); -#endif /* PML_BFO */ - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_RNDV, - sendreq->req_send.req_base.req_proc); - - /* update lengths with number of bytes actually packed */ - segment->seg_len = sizeof(mca_pml_bfo_rendezvous_hdr_t); - - /* first fragment of a long message */ - des->des_cbfunc = mca_pml_bfo_rndv_completion; - need_local_cb = true; - - /* wait for ack and completion */ - sendreq->req_state = 2; - } - - des->des_cbdata = sendreq; - - /* send */ - rc = mca_bml_base_send(bml_btl, des, hdr->hdr_common.hdr_type); - if( OPAL_LIKELY( rc >= 0 ) ) { - if( OPAL_LIKELY( 1 == rc ) && (true == need_local_cb)) { - mca_pml_bfo_rndv_completion_request( bml_btl, sendreq, 0 ); - } -#if PML_BFO - if (MCA_PML_BFO_HDR_TYPE_RNDV == hdr->hdr_common.hdr_type) { - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - sendreq->req_events++; - } - } -#endif /* PML_BFO */ - return OMPI_SUCCESS; - } - mca_bml_base_free(bml_btl, des); - if (NULL != src) { - mca_bml_base_free (bml_btl, src); - } - - return rc; -} - - -/** - * Rendezvous is required. Not doing rdma so eager send up to - * the btls eager limit. - */ - -int mca_pml_bfo_send_request_start_rndv( mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size, - int flags ) -{ - mca_btl_base_descriptor_t* des; - mca_btl_base_segment_t* segment; - mca_pml_bfo_hdr_t* hdr; - int rc; - - /* prepare descriptor */ - if(size == 0) { - mca_bml_base_alloc( bml_btl, - &des, - MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_rendezvous_hdr_t), - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ); - } else { - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - mca_bml_base_prepare_src( bml_btl, - NULL, - &sendreq->req_send.req_base.req_convertor, - MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_rendezvous_hdr_t), - &size, - MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP, - &des ); - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - } - - if( OPAL_UNLIKELY(NULL == des) ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - segment = des->des_local; - - /* build hdr */ - hdr = (mca_pml_bfo_hdr_t*)segment->seg_addr.pval; - hdr->hdr_common.hdr_flags = flags; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_RNDV; - hdr->hdr_match.hdr_ctx = sendreq->req_send.req_base.req_comm->c_contextid; - hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank; - hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag; - hdr->hdr_match.hdr_seq = (uint16_t)sendreq->req_send.req_base.req_sequence; - hdr->hdr_rndv.hdr_msg_length = sendreq->req_send.req_bytes_packed; - hdr->hdr_rndv.hdr_src_req.pval = sendreq; -#if PML_BFO - MCA_PML_BFO_CHECK_FOR_RNDV_RESTART(hdr, sendreq, "RNDV"); -#endif /* PML_BFO */ - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_RNDV, - sendreq->req_send.req_base.req_proc); - - /* first fragment of a long message */ - des->des_cbdata = sendreq; - des->des_cbfunc = mca_pml_bfo_rndv_completion; - - /* wait for ack and completion */ - sendreq->req_state = 2; - - /* send */ - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_RNDV); - if( OPAL_LIKELY( rc >= 0 ) ) { - if( OPAL_LIKELY( 1 == rc ) ) { - mca_pml_bfo_rndv_completion_request( bml_btl, sendreq, size ); - } -#if PML_BFO - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - sendreq->req_events++; - } -#endif /* PML_BFO */ - return OMPI_SUCCESS; - } - mca_bml_base_free(bml_btl, des ); - return rc; -} - -void mca_pml_bfo_send_request_copy_in_out( mca_pml_bfo_send_request_t *sendreq, - uint64_t send_offset, - uint64_t send_length ) -{ - mca_pml_bfo_send_range_t *sr; - ompi_free_list_item_t *i; - mca_bml_base_endpoint_t* bml_endpoint = sendreq->req_endpoint; - int num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_send); - int n; - double weight_total = 0; - - if( OPAL_UNLIKELY(0 == send_length) ) - return; - - OMPI_FREE_LIST_WAIT_MT(&mca_pml_bfo.send_ranges, i); - - sr = (mca_pml_bfo_send_range_t*)i; - - sr->range_send_offset = send_offset; - sr->range_send_length = send_length; - sr->range_btl_idx = 0; - - for(n = 0; n < num_btls && n < mca_pml_bfo.max_send_per_range; n++) { - sr->range_btls[n].bml_btl = - mca_bml_base_btl_array_get_next(&bml_endpoint->btl_send); - weight_total += sr->range_btls[n].bml_btl->btl_weight; - } - - sr->range_btl_cnt = n; - mca_pml_bfo_calc_weighted_length(sr->range_btls, n, send_length, - weight_total); - - OPAL_THREAD_LOCK(&sendreq->req_send_range_lock); - opal_list_append(&sendreq->req_send_ranges, (opal_list_item_t*)sr); - OPAL_THREAD_UNLOCK(&sendreq->req_send_range_lock); -} - -static inline mca_pml_bfo_send_range_t * -get_send_range_nolock(mca_pml_bfo_send_request_t* sendreq) -{ - opal_list_item_t *item; - - item = opal_list_get_first(&sendreq->req_send_ranges); - - if(opal_list_get_end(&sendreq->req_send_ranges) == item) - return NULL; - - return (mca_pml_bfo_send_range_t*)item; -} - -static inline mca_pml_bfo_send_range_t * -get_send_range(mca_pml_bfo_send_request_t* sendreq) -{ - mca_pml_bfo_send_range_t *range; - - OPAL_THREAD_LOCK(&sendreq->req_send_range_lock); - range = get_send_range_nolock(sendreq); - OPAL_THREAD_UNLOCK(&sendreq->req_send_range_lock); - - return range; -} - -static inline mca_pml_bfo_send_range_t * -get_next_send_range(mca_pml_bfo_send_request_t* sendreq, - mca_pml_bfo_send_range_t *range) -{ - OPAL_THREAD_LOCK(&sendreq->req_send_range_lock); - opal_list_remove_item(&sendreq->req_send_ranges, (opal_list_item_t *)range); - OMPI_FREE_LIST_RETURN_MT(&mca_pml_bfo.send_ranges, &range->base); - range = get_send_range_nolock(sendreq); - OPAL_THREAD_UNLOCK(&sendreq->req_send_range_lock); - - return range; -} - -/** - * Schedule pipeline of send descriptors for the given request. - * Up to the rdma threshold. If this is a send based protocol, - * the rdma threshold is the end of the message. Otherwise, schedule - * fragments up to the threshold to overlap initial registration/setup - * costs of the rdma. Only one thread can be inside this function. - */ - -int -mca_pml_bfo_send_request_schedule_once(mca_pml_bfo_send_request_t* sendreq) -{ - size_t prev_bytes_remaining = 0; - mca_pml_bfo_send_range_t *range; - int num_fail = 0; - - /* check pipeline_depth here before attempting to get any locks */ - if(true == sendreq->req_throttle_sends && - sendreq->req_pipeline_depth >= mca_pml_bfo.send_pipeline_depth) - return OMPI_SUCCESS; - - range = get_send_range(sendreq); - - while(range && (false == sendreq->req_throttle_sends || - sendreq->req_pipeline_depth < mca_pml_bfo.send_pipeline_depth)) { - mca_pml_bfo_frag_hdr_t* hdr; - mca_btl_base_descriptor_t* des; - int rc, btl_idx; - size_t size, offset, data_remaining = 0; - mca_bml_base_btl_t* bml_btl; - - assert(range->range_send_length != 0); -#if PML_BFO - MCA_PML_BFO_CHECK_FOR_REMOVED_BTL(sendreq, range); -#endif /* PML_BFO */ - - if(prev_bytes_remaining == range->range_send_length) - num_fail++; - else - num_fail = 0; - - prev_bytes_remaining = range->range_send_length; - - if( OPAL_UNLIKELY(num_fail == range->range_btl_cnt) ) { - assert(sendreq->req_pending == MCA_PML_BFO_SEND_PENDING_NONE); - add_request_to_send_pending(sendreq, - MCA_PML_BFO_SEND_PENDING_SCHEDULE, true); - /* Note that request remains locked. send_request_process_pending() - * function will call shedule_exclusive() directly without taking - * the lock */ - return OMPI_ERR_OUT_OF_RESOURCE; - } - -cannot_pack: - do { - btl_idx = range->range_btl_idx; - if(++range->range_btl_idx == range->range_btl_cnt) - range->range_btl_idx = 0; - } while(!range->range_btls[btl_idx].length); - - bml_btl = range->range_btls[btl_idx].bml_btl; - /* If there is a remaining data from another BTL that was too small - * for converter to pack then send it through another BTL */ - range->range_btls[btl_idx].length += data_remaining; - size = range->range_btls[btl_idx].length; - - /* makes sure that we don't exceed BTL max send size */ - if(bml_btl->btl->btl_max_send_size != 0) { - size_t max_send_size = bml_btl->btl->btl_max_send_size - - sizeof(mca_pml_bfo_frag_hdr_t); - - if (size > max_send_size) { - size = max_send_size; - } - } - - /* pack into a descriptor */ - offset = (size_t)range->range_send_offset; - opal_convertor_set_position(&sendreq->req_send.req_base.req_convertor, - &offset); - range->range_send_offset = (uint64_t)offset; - - data_remaining = size; - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_defined, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - mca_bml_base_prepare_src(bml_btl, NULL, - &sendreq->req_send.req_base.req_convertor, - MCA_BTL_NO_ORDER, - sizeof(mca_pml_bfo_frag_hdr_t), - &size, MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK, &des); - MEMCHECKER( - memchecker_call(&opal_memchecker_base_mem_noaccess, - sendreq->req_send.req_base.req_addr, - sendreq->req_send.req_base.req_count, - sendreq->req_send.req_base.req_datatype); - ); - - if( OPAL_UNLIKELY(des == NULL || size == 0) ) { - if(des) { - /* Converter can't pack this chunk. Append to another chunk - * from other BTL */ - mca_bml_base_free(bml_btl, des); - range->range_btls[btl_idx].length -= data_remaining; - goto cannot_pack; - } - continue; - } - - des->des_cbfunc = mca_pml_bfo_frag_completion; - des->des_cbdata = sendreq; - - /* setup header */ - hdr = (mca_pml_bfo_frag_hdr_t*)des->des_local->seg_addr.pval; - hdr->hdr_common.hdr_flags = 0; - hdr->hdr_common.hdr_type = MCA_PML_BFO_HDR_TYPE_FRAG; - hdr->hdr_frag_offset = range->range_send_offset; - hdr->hdr_src_req.pval = sendreq; - hdr->hdr_dst_req = sendreq->req_recv; - - bfo_hdr_hton(hdr, MCA_PML_BFO_HDR_TYPE_FRAG, - sendreq->req_send.req_base.req_proc); - -#if OMPI_WANT_PERUSE - PERUSE_TRACE_COMM_OMPI_EVENT(PERUSE_COMM_REQ_XFER_CONTINUE, - &(sendreq->req_send.req_base), size, PERUSE_SEND); -#endif /* OMPI_WANT_PERUSE */ - - /* initiate send - note that this may complete before the call returns */ - rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_FRAG); - if( OPAL_LIKELY(rc >= 0) ) { - /* update state */ - range->range_btls[btl_idx].length -= size; - range->range_send_length -= size; - range->range_send_offset += size; - OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_pipeline_depth, 1); - if(range->range_send_length == 0) { - range = get_next_send_range(sendreq, range); - prev_bytes_remaining = 0; - } -#if PML_BFO - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - sendreq->req_events++; - } -#endif /* PML_BFO */ - } else { - mca_bml_base_free(bml_btl,des); - } - } - - return OMPI_SUCCESS; -} - - -/** - * An RDMA put operation has completed: - * (1) Update request status and if required set completed - * (2) Send FIN control message to the destination - */ - -static void mca_pml_bfo_put_completion( mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* ep, - struct mca_btl_base_descriptor_t* des, - int status ) -{ - mca_pml_bfo_rdma_frag_t* frag = (mca_pml_bfo_rdma_frag_t*)des->des_cbdata; - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)frag->rdma_req; - mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context; - - /* check completion status */ - if( OPAL_UNLIKELY(OMPI_SUCCESS != status) ) { -#if PML_BFO - sendreq->req_error++; -#else /* PML_BFO */ - /* TSW - FIX */ - OMPI_ERROR_LOG(status); - ompi_rte_abort(-1, NULL); -#endif /* PML_BFO */ - } -#if PML_BFO - sendreq->req_events--; - MCA_PML_BFO_PUT_COMPLETION_SENDREQ_ERROR_CHECK(sendreq, status, btl); - MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, "RDMA write"); -#endif /* PML_BFO */ - - mca_pml_bfo_send_fin(sendreq->req_send.req_base.req_proc, - bml_btl, - frag->rdma_hdr.hdr_rdma.hdr_des, -#if PML_BFO - des->order, 0, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_send.req_base.req_comm->c_my_rank); -#else /* PML_BFO */ - des->order, 0); -#endif /* PML_BFO */ - - /* check for request completion */ - OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, frag->rdma_length); - - send_request_pml_complete_check(sendreq); - - MCA_PML_BFO_RDMA_FRAG_RETURN(frag); - - MCA_PML_BFO_PROGRESS_PENDING(bml_btl); -} - -int mca_pml_bfo_send_request_put_frag( mca_pml_bfo_rdma_frag_t* frag ) -{ - mca_mpool_base_registration_t* reg = NULL; - mca_bml_base_btl_t* bml_btl = frag->rdma_bml; - mca_btl_base_descriptor_t* des; - size_t save_size = frag->rdma_length; - int rc; - - /* setup descriptor */ - mca_bml_base_prepare_src( bml_btl, - reg, - &frag->convertor, - MCA_BTL_NO_ORDER, - 0, - &frag->rdma_length, - MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | - MCA_BTL_DES_FLAGS_PUT, - &des ); - - if( OPAL_UNLIKELY(NULL == des) ) { - if(frag->retries < mca_pml_bfo.rdma_put_retries_limit) { - size_t offset = (size_t)frag->rdma_hdr.hdr_rdma.hdr_rdma_offset; - frag->rdma_length = save_size; - opal_convertor_set_position(&frag->convertor, &offset); - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.rdma_pending, (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - } else { - mca_pml_bfo_send_request_t *sendreq = - (mca_pml_bfo_send_request_t*)frag->rdma_req; - - /* tell receiver to unregister memory */ - mca_pml_bfo_send_fin(sendreq->req_send.req_base.req_proc, - bml_btl, frag->rdma_hdr.hdr_rdma.hdr_des, -#if PML_BFO - MCA_BTL_NO_ORDER, 1, (uint16_t)sendreq->req_send.req_base.req_sequence, - sendreq->req_restartseq, sendreq->req_send.req_base.req_comm->c_contextid, - sendreq->req_send.req_base.req_comm->c_my_rank); -#else /* PML_BFO */ - MCA_BTL_NO_ORDER, 1); -#endif /* PML_BFO */ - - /* send fragment by copy in/out */ - mca_pml_bfo_send_request_copy_in_out(sendreq, - frag->rdma_hdr.hdr_rdma.hdr_rdma_offset, frag->rdma_length); - /* if a pointer to a receive request is not set it means that - * ACK was not yet received. Don't schedule sends before ACK */ - if(NULL != sendreq->req_recv.pval) - mca_pml_bfo_send_request_schedule(sendreq); - } - return OMPI_ERR_OUT_OF_RESOURCE; - } - - des->des_remote = (mca_btl_base_segment_t *) frag->rdma_segs; - des->des_remote_count = frag->rdma_hdr.hdr_rdma.hdr_seg_cnt; - des->des_cbfunc = mca_pml_bfo_put_completion; - des->des_cbdata = frag; - - PERUSE_TRACE_COMM_OMPI_EVENT( PERUSE_COMM_REQ_XFER_CONTINUE, - &(((mca_pml_bfo_send_request_t*)frag->rdma_req)->req_send.req_base), save_size, PERUSE_SEND ); - - rc = mca_bml_base_put(bml_btl, des); - if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { - mca_bml_base_free(bml_btl, des); - frag->rdma_length = save_size; - if(OMPI_ERR_OUT_OF_RESOURCE == rc) { - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - opal_list_append(&mca_pml_bfo.rdma_pending, (opal_list_item_t*)frag); - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - return OMPI_ERR_OUT_OF_RESOURCE; - } else { - /* TSW - FIX */ - OMPI_ERROR_LOG(rc); - ompi_rte_abort(-1, NULL); - } - } -#if PML_BFO - if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) { - ((mca_pml_bfo_send_request_t*)frag->rdma_req)->req_events++; - } -#endif /* PML_BFO */ - return OMPI_SUCCESS; -} - -/** - * Receiver has scheduled an RDMA operation: - * (1) Allocate an RDMA fragment to maintain the state of the operation - * (2) Call BTL prepare_src to pin/prepare source buffers - * (3) Queue the RDMA put - */ - -void mca_pml_bfo_send_request_put( mca_pml_bfo_send_request_t* sendreq, - mca_btl_base_module_t* btl, - mca_pml_bfo_rdma_hdr_t* hdr ) -{ - mca_bml_base_endpoint_t *bml_endpoint = sendreq->req_endpoint; - mca_pml_bfo_rdma_frag_t* frag; - size_t i, size = 0; - - if(hdr->hdr_common.hdr_flags & MCA_PML_BFO_HDR_TYPE_ACK) { - OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, -1); - } -#if PML_BFO - MCA_PML_BFO_VERIFY_SENDREQ_REQ_STATE_VALUE(sendreq); - sendreq->req_recv = hdr->hdr_dst_req; /* only needed once, but it is OK */ -#endif /* PML_BFO */ - - MCA_PML_BFO_RDMA_FRAG_ALLOC(frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - /* TSW - FIX */ - OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE); - ompi_rte_abort(-1, NULL); - } - - assert (btl->btl_seg_size * hdr->hdr_seg_cnt <= sizeof (frag->rdma_segs)); - - /* setup fragment */ - memmove (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt); - - for( i = 0; i < hdr->hdr_seg_cnt; i++ ) { - mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *) ((uintptr_t)(frag->rdma_segs) + i * btl->btl_seg_size); - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - if ((sendreq->req_send.req_base.req_proc->super.proc_arch & OPAL_ARCH_ISBIGENDIAN) != - (ompi_proc_local()->super.proc_arch & OPAL_ARCH_ISBIGENDIAN)) { - size += opal_swap_bytes4(seg->seg_len); - } else -#endif - { - size += seg->seg_len; - } - } - - frag->rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl); -#if PML_BFO - MCA_PML_BFO_CHECK_FOR_REMOVED_BML(sendreq, frag, btl); - frag->rdma_btl = btl; /* in case frag ends up on pending */ -#endif /* PML_BFO */ - frag->rdma_hdr.hdr_rdma = *hdr; - frag->rdma_req = sendreq; - frag->rdma_ep = bml_endpoint; - frag->rdma_length = size; - frag->rdma_state = MCA_PML_BFO_RDMA_PUT; - frag->reg = NULL; - frag->retries = 0; - - /* lookup the corresponding registration */ - for(i=0; ireq_rdma_cnt; i++) { - if(sendreq->req_rdma[i].bml_btl == frag->rdma_bml) { - frag->reg = sendreq->req_rdma[i].btl_reg; - break; - } - } - - /* RDMA writes may proceed in parallel to send and to each other, so - * create clone of the convertor for each RDMA fragment - */ - size = hdr->hdr_rdma_offset; - opal_convertor_clone_with_position(&sendreq->req_send.req_base.req_convertor, - &frag->convertor, 0, &size); - - mca_pml_bfo_send_request_put_frag(frag); -} - diff --git a/ompi/mca/pml/bfo/pml_bfo_sendreq.h b/ompi/mca/pml/bfo/pml_bfo_sendreq.h deleted file mode 100644 index 170512ffe3..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_sendreq.h +++ /dev/null @@ -1,499 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011-2012 Los Alamos National Security, LLC. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OMPI_PML_BFO_SEND_REQUEST_H -#define OMPI_PML_BFO_SEND_REQUEST_H - -#include "opal/mca/btl/btl.h" -#include "opal/mca/mpool/base/base.h" -#include "ompi/mca/pml/base/pml_base_sendreq.h" -#include "pml_bfo_comm.h" -#include "pml_bfo_hdr.h" -#include "pml_bfo_rdma.h" -#include "pml_bfo_rdmafrag.h" -#include "ompi/mca/bml/bml.h" - -BEGIN_C_DECLS - -typedef enum { - MCA_PML_BFO_SEND_PENDING_NONE, - MCA_PML_BFO_SEND_PENDING_SCHEDULE, - MCA_PML_BFO_SEND_PENDING_START -} mca_pml_bfo_send_pending_t; - -struct mca_pml_bfo_send_request_t { - mca_pml_base_send_request_t req_send; - mca_bml_base_endpoint_t* req_endpoint; - opal_ptr_t req_recv; -#if PML_BFO - int32_t req_events; /* number of outstanding events on request */ - int32_t req_restartseq; /* sequence number of restarted request */ - int32_t req_restart; /* state of restarted request */ - int32_t req_error; /* non-zero when error has occurred on request */ -#endif /* PML_BFO */ - int32_t req_state; - int32_t req_lock; - bool req_throttle_sends; - size_t req_pipeline_depth; - size_t req_bytes_delivered; - uint32_t req_rdma_cnt; - mca_pml_bfo_send_pending_t req_pending; - opal_mutex_t req_send_range_lock; - opal_list_t req_send_ranges; - mca_pml_bfo_com_btl_t req_rdma[1]; -}; -typedef struct mca_pml_bfo_send_request_t mca_pml_bfo_send_request_t; - -OBJ_CLASS_DECLARATION(mca_pml_bfo_send_request_t); - -struct mca_pml_bfo_send_range_t { - opal_free_list_item_t base; - uint64_t range_send_offset; - uint64_t range_send_length; - int range_btl_idx; - int range_btl_cnt; - mca_pml_bfo_com_btl_t range_btls[1]; -}; -typedef struct mca_pml_bfo_send_range_t mca_pml_bfo_send_range_t; -OBJ_CLASS_DECLARATION(mca_pml_bfo_send_range_t); - -static inline bool lock_send_request(mca_pml_bfo_send_request_t *sendreq) -{ - return OPAL_THREAD_ADD_FETCH32(&sendreq->req_lock, 1) == 1; -} - -static inline bool unlock_send_request(mca_pml_bfo_send_request_t *sendreq) -{ - return OPAL_THREAD_ADD_FETCH32(&sendreq->req_lock, -1) == 0; -} - -static inline void -add_request_to_send_pending(mca_pml_bfo_send_request_t* sendreq, - const mca_pml_bfo_send_pending_t type, - const bool append) -{ - opal_list_item_t *item = (opal_list_item_t*)sendreq; - - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - sendreq->req_pending = type; - if(append) - opal_list_append(&mca_pml_bfo.send_pending, item); - else - opal_list_prepend(&mca_pml_bfo.send_pending, item); - - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); -} - -static inline mca_pml_bfo_send_request_t* -get_request_from_send_pending(mca_pml_bfo_send_pending_t *type) -{ - mca_pml_bfo_send_request_t *sendreq; - - OPAL_THREAD_LOCK(&mca_pml_bfo.lock); - sendreq = (mca_pml_bfo_send_request_t*) - opal_list_remove_first(&mca_pml_bfo.send_pending); - if(sendreq) { - *type = sendreq->req_pending; - sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE; - } - OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock); - - return sendreq; -} - -#define MCA_PML_BFO_SEND_REQUEST_ALLOC( comm, \ - dst, \ - sendreq) \ - { \ - ompi_proc_t *proc = ompi_comm_peer_lookup( comm, dst ); \ - opal_free_list_item_t* item; \ - \ - sendreq = NULL; \ - if( OPAL_LIKELY(NULL != proc) ) { \ - OPAL_FREE_LIST_WAIT_MT(&mca_pml_base_send_requests, item); \ - sendreq = (mca_pml_bfo_send_request_t*)item; \ - sendreq->req_send.req_base.req_proc = proc; \ - } \ - } - - -#define MCA_PML_BFO_SEND_REQUEST_INIT( sendreq, \ - buf, \ - count, \ - datatype, \ - dst, \ - tag, \ - comm, \ - sendmode, \ - persistent) \ - { \ - MCA_PML_BASE_SEND_REQUEST_INIT(&sendreq->req_send, \ - buf, \ - count, \ - datatype, \ - dst, \ - tag, \ - comm, \ - sendmode, \ - persistent, \ - 0); /* convertor_flags */ \ - (sendreq)->req_recv.pval = NULL; \ - } - - -static inline void mca_pml_bfo_free_rdma_resources(mca_pml_bfo_send_request_t* sendreq) -{ - size_t r; - - /* return mpool resources */ - for(r = 0; r < sendreq->req_rdma_cnt; r++) { - struct mca_btl_base_registration_handle_t* handle = sendreq->req_rdma[r].btl_reg; - mca_bml_base_btl_t *bml_btl = sendreq->req_rdma[r].bml_btl; - - if( NULL != handle ) { - mca_bml_base_deregister_mem (bml_btl, handle); - sendreq->req_rdma[r].btl_reg = NULL; - } - } - sendreq->req_rdma_cnt = 0; -} - - -/** - * Start a send request. - */ - -#define MCA_PML_BFO_SEND_REQUEST_START(sendreq, rc) \ - do { \ - rc = mca_pml_bfo_send_request_start(sendreq); \ - } while (0) - - -/* - * Mark a send request as completed at the MPI level. - */ - -#define MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq, with_signal) \ -do { \ - (sendreq)->req_send.req_base.req_ompi.req_status.MPI_SOURCE = \ - (sendreq)->req_send.req_base.req_comm->c_my_rank; \ - (sendreq)->req_send.req_base.req_ompi.req_status.MPI_TAG = \ - (sendreq)->req_send.req_base.req_tag; \ - (sendreq)->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS; \ - (sendreq)->req_send.req_base.req_ompi.req_status._ucount = \ - (sendreq)->req_send.req_bytes_packed; \ - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE, \ - &(sendreq->req_send.req_base), PERUSE_SEND); \ - \ - ompi_request_complete( &((sendreq)->req_send.req_base.req_ompi), (with_signal) ); \ -} while(0) - -/* - * Release resources associated with a request - */ - -#define MCA_PML_BFO_SEND_REQUEST_RETURN(sendreq) \ - do { \ - MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send)); \ - OPAL_FREE_LIST_RETURN( &mca_pml_base_send_requests, \ - (opal_free_list_item_t*)sendreq); \ - } while(0) - - -/* - * The PML has completed a send request. Note that this request - * may have been orphaned by the user or have already completed - * at the MPI level. - * This function will never be called directly from the upper level, as it - * should only be an internal call to the PML. - * - */ -static inline void -send_request_pml_complete(mca_pml_bfo_send_request_t *sendreq) -{ - if(false == sendreq->req_send.req_base.req_pml_complete) { - if(sendreq->req_send.req_bytes_packed > 0) { - PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END, - &(sendreq->req_send.req_base), PERUSE_SEND); - } - - /* return mpool resources */ - mca_pml_bfo_free_rdma_resources(sendreq); - - if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED && - sendreq->req_send.req_addr != sendreq->req_send.req_base.req_addr) { - mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); - } - - sendreq->req_send.req_base.req_pml_complete = true; - - if( !REQUEST_COMPLETE( &((sendreq->req_send).req_base.req_ompi)) ) { - /* Should only be called for long messages (maybe synchronous) */ - MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq, true); - } else { - if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) { - ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST); - } - } -#if PML_BFO - sendreq->req_send.req_base.req_sequence -= 100; -#endif /* PML_BFO */ - - if(true == sendreq->req_send.req_base.req_free_called) { - MCA_PML_BFO_SEND_REQUEST_RETURN(sendreq); - } - } -} - -/* returns true if request was completed on PML level */ -static inline bool -send_request_pml_complete_check(mca_pml_bfo_send_request_t *sendreq) -{ -#if OPAL_ENABLE_MULTI_THREADS - opal_atomic_rmb(); -#endif - /* if no more events are expected for the request and the whole message is - * already sent and send fragment scheduling isn't running in another - * thread then complete the request on PML level. From now on, if user - * called free on this request, the request structure can be reused for - * another request or if the request is persistent it can be restarted */ - if(sendreq->req_state == 0 && - sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed - && lock_send_request(sendreq)) { - send_request_pml_complete(sendreq); - return true; - } - - return false; -} - -/** - * Schedule additional fragments - */ -int -mca_pml_bfo_send_request_schedule_once(mca_pml_bfo_send_request_t*); - -static inline int -mca_pml_bfo_send_request_schedule_exclusive(mca_pml_bfo_send_request_t* sendreq) -{ - int rc; - do { - rc = mca_pml_bfo_send_request_schedule_once(sendreq); - if(rc == OMPI_ERR_OUT_OF_RESOURCE) - break; - } while(!unlock_send_request(sendreq)); - - if(OMPI_SUCCESS == rc) - send_request_pml_complete_check(sendreq); - - return rc; -} - -static inline void -mca_pml_bfo_send_request_schedule(mca_pml_bfo_send_request_t* sendreq) -{ - /* - * Only allow one thread in this routine for a given request. - * However, we cannot block callers on a mutex, so simply keep track - * of the number of times the routine has been called and run through - * the scheduling logic once for every call. - */ - - if(!lock_send_request(sendreq)) - return; - - mca_pml_bfo_send_request_schedule_exclusive(sendreq); -} - -#if OPAL_CUDA_SUPPORT -int mca_pml_bfo_send_request_start_cuda( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size); -#endif /* OPAL_CUDA_SUPPORT */ - -/** - * Start the specified request - */ - -int mca_pml_bfo_send_request_start_buffered( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size); - -int mca_pml_bfo_send_request_start_copy( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size); - -int mca_pml_bfo_send_request_start_prepare( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size); - -int mca_pml_bfo_send_request_start_rdma( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size); - -int mca_pml_bfo_send_request_start_rndv( - mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl, - size_t size, - int flags); - -static inline int -mca_pml_bfo_send_request_start_btl( mca_pml_bfo_send_request_t* sendreq, - mca_bml_base_btl_t* bml_btl ) -{ - size_t size = sendreq->req_send.req_bytes_packed; - mca_btl_base_module_t* btl = bml_btl->btl; - size_t eager_limit = btl->btl_eager_limit - sizeof(mca_pml_bfo_hdr_t); - int rc; - - if( OPAL_LIKELY(size <= eager_limit) ) { - switch(sendreq->req_send.req_send_mode) { - case MCA_PML_BASE_SEND_SYNCHRONOUS: - rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0); - break; - case MCA_PML_BASE_SEND_BUFFERED: - rc = mca_pml_bfo_send_request_start_copy(sendreq, bml_btl, size); - break; - case MCA_PML_BASE_SEND_COMPLETE: - rc = mca_pml_bfo_send_request_start_prepare(sendreq, bml_btl, size); - break; - default: - if (size != 0 && bml_btl->btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) { - rc = mca_pml_bfo_send_request_start_prepare(sendreq, bml_btl, size); - } else { - rc = mca_pml_bfo_send_request_start_copy(sendreq, bml_btl, size); - } - break; - } - } else { - size = eager_limit; - if(OPAL_UNLIKELY(btl->btl_rndv_eager_limit < eager_limit)) - size = btl->btl_rndv_eager_limit; - if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { - rc = mca_pml_bfo_send_request_start_buffered(sendreq, bml_btl, size); - } else if - (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) { - unsigned char *base; - opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base ); - - if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_bfo_rdma_btls( - sendreq->req_endpoint, - base, - sendreq->req_send.req_bytes_packed, - sendreq->req_rdma))) { - rc = mca_pml_bfo_send_request_start_rdma(sendreq, bml_btl, - sendreq->req_send.req_bytes_packed); - if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { - mca_pml_bfo_free_rdma_resources(sendreq); - } - } else { - rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, - MCA_PML_BFO_HDR_FLAGS_CONTIG); - } - } else { -#if OPAL_CUDA_SUPPORT - if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) { - return mca_pml_bfo_send_request_start_cuda(sendreq, bml_btl, size); - } -#endif /* OPAL_CUDA_SUPPORT */ - rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0); - } - } - - return rc; -} - -static inline int -mca_pml_bfo_send_request_start( mca_pml_bfo_send_request_t* sendreq ) -{ - mca_pml_bfo_comm_t* comm = sendreq->req_send.req_base.req_comm->c_pml_comm; - mca_bml_base_endpoint_t* endpoint = (mca_bml_base_endpoint_t*) - sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; - size_t i; - - if( OPAL_UNLIKELY(endpoint == NULL) ) { - return OMPI_ERR_UNREACH; - } - - sendreq->req_endpoint = endpoint; - sendreq->req_state = 0; - sendreq->req_lock = 0; - sendreq->req_pipeline_depth = 0; - sendreq->req_bytes_delivered = 0; - sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE; - sendreq->req_send.req_base.req_sequence = OPAL_THREAD_ADD_FETCH32( - &comm->procs[sendreq->req_send.req_base.req_peer].send_sequence,1); -#if PML_BFO - sendreq->req_restartseq = 0; /* counts up restarts */ - sendreq->req_restart = 0; /* reset in case we restart again */ - sendreq->req_error = 0; /* clear error state */ - sendreq->req_events = 0; /* clear events, probably 0 anyways */ -#endif /* PML_BFO */ - - MCA_PML_BASE_SEND_START( &sendreq->req_send.req_base ); - - for(i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) { - mca_bml_base_btl_t* bml_btl; - int rc; - - /* select a btl */ - bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager); - rc = mca_pml_bfo_send_request_start_btl(sendreq, bml_btl); - if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc) ) - return rc; - } - add_request_to_send_pending(sendreq, MCA_PML_BFO_SEND_PENDING_START, true); - - return OMPI_SUCCESS; -} - -/** - * Initiate a put scheduled by the receiver. - */ - -void mca_pml_bfo_send_request_put( mca_pml_bfo_send_request_t* sendreq, - mca_btl_base_module_t* btl, - mca_pml_bfo_rdma_hdr_t* hdr ); - -int mca_pml_bfo_send_request_put_frag(mca_pml_bfo_rdma_frag_t* frag); - -/* This function tries to continue sendreq that was stuck because of resource - * unavailability. A sendreq may be added to send_pending list if there is no - * resource to send initial packet or there is not resource to schedule data - * for sending. The reason the sendreq was added to the list is stored inside - * sendreq struct and appropriate operation is retried when resource became - * available. bml_btl passed to the function doesn't represents sendreq - * destination, it represents BTL on which resource was freed, so only this BTL - * should be considered for sending packets */ -void mca_pml_bfo_send_request_process_pending(mca_bml_base_btl_t *bml_btl); - -void mca_pml_bfo_send_request_copy_in_out(mca_pml_bfo_send_request_t *sendreq, - uint64_t send_offset, uint64_t send_length); - -END_C_DECLS - -#endif /* OMPI_PML_BFO_SEND_REQUEST_H */ diff --git a/ompi/mca/pml/bfo/pml_bfo_start.c b/ompi/mca/pml/bfo/pml_bfo_start.c deleted file mode 100644 index 93e6216c92..0000000000 --- a/ompi/mca/pml/bfo/pml_bfo_start.c +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "pml_bfo.h" -#include "pml_bfo_recvreq.h" -#include "pml_bfo_sendreq.h" -#include "ompi/memchecker.h" - - -int mca_pml_bfo_start(size_t count, ompi_request_t** requests) -{ - int rc; - size_t i; - bool reuse_old_request = true; - - for(i=0; ireq_type) { - continue; - } - - /* If the persistent request is currently active - obtain the - * request lock and verify the status is incomplete. if the - * pml layer has not completed the request - mark the request - * as free called - so that it will be freed when the request - * completes - and create a new request. - */ - -#if OPAL_ENABLE_MULTI_THREADS - opal_atomic_rmb(); -#endif - reuse_old_request = true; - switch(pml_request->req_ompi.req_state) { - case OMPI_REQUEST_INACTIVE: - if(pml_request->req_pml_complete == true) - break; - /* otherwise fall through */ - case OMPI_REQUEST_ACTIVE: { - - ompi_request_t *request; - if (pml_request->req_pml_complete == false) { - /* free request after it completes */ - pml_request->req_free_called = true; - } else { - /* can reuse the existing request */ - break; - } - - reuse_old_request = false; - /* allocate a new request */ - switch(pml_request->req_type) { - case MCA_PML_REQUEST_SEND: { - mca_pml_base_send_mode_t sendmode = - ((mca_pml_base_send_request_t*)pml_request)->req_send_mode; - rc = mca_pml_bfo_isend_init( - pml_request->req_addr, - pml_request->req_count, - pml_request->req_datatype, - pml_request->req_peer, - pml_request->req_tag, - sendmode, - pml_request->req_comm, - &request); - break; - } - case MCA_PML_REQUEST_RECV: - rc = mca_pml_bfo_irecv_init( - pml_request->req_addr, - pml_request->req_count, - pml_request->req_datatype, - pml_request->req_peer, - pml_request->req_tag, - pml_request->req_comm, - &request); - break; - default: - rc = OMPI_ERR_REQUEST; - break; - } - if(OMPI_SUCCESS != rc) - return rc; - pml_request = (mca_pml_base_request_t*)request; - requests[i] = request; - break; - } - default: - return OMPI_ERR_REQUEST; - } - - /* start the request */ - switch(pml_request->req_type) { - case MCA_PML_REQUEST_SEND: - { - mca_pml_bfo_send_request_t* sendreq = (mca_pml_bfo_send_request_t*)pml_request; - MEMCHECKER( - memchecker_call(&opal_memchecker_base_isdefined, - pml_request->req_addr, pml_request->req_count, - pml_request->req_datatype); - ); - if( reuse_old_request && (sendreq->req_send.req_bytes_packed != 0) ) { - size_t offset = 0; - /** - * Reset the convertor in case we're dealing with the original - * request, which when completed do not reset the convertor. - */ - opal_convertor_set_position( &sendreq->req_send.req_base.req_convertor, - &offset ); - } - MCA_PML_BFO_SEND_REQUEST_START(sendreq, rc); - if(rc != OMPI_SUCCESS) - return rc; - break; - } - case MCA_PML_REQUEST_RECV: - { - mca_pml_bfo_recv_request_t* recvreq = (mca_pml_bfo_recv_request_t*)pml_request; - MCA_PML_BFO_RECV_REQUEST_START(recvreq); - break; - } - default: - return OMPI_ERR_REQUEST; - } - } - return OMPI_SUCCESS; -} - diff --git a/ompi/mca/pml/bfo/post_configure.sh b/ompi/mca/pml/bfo/post_configure.sh deleted file mode 100644 index 77a7d52608..0000000000 --- a/ompi/mca/pml/bfo/post_configure.sh +++ /dev/null @@ -1 +0,0 @@ -DIRECT_CALL_HEADER="ompi/mca/pml/bfo/pml_bfo.h"