PostOldWindowExchange Subroutine

  subroutine PostOldWindowExchange(decomp,perElem,nvar,nLocalOld,uLocal,winFirst,winLast, &
                                   wFirst,wLast,uWin,requests,msgCount, &
                                   nBytesRecv,nBytesSent,nElemRemote)
    !! Post every message of this rank's migration schedule and return without waiting: the
    !! receives that fill the remote runs of its window, then the sends that serve the runs of its
    !! old range that other ranks' windows need. Receives are posted before sends, as everywhere
    !! else in SELF.
    !!
    !! Neither buffer is touched other than by MPI, so this is also the device-resident form: a
    !! GPU backend passes descriptors over device memory and the received runs land there
    !! directly. The caller supplies the request array (sized for the worst case of every rank
    !! being a peer) and pairs this with FinishOldWindowExchange.
    !!
    !! The whole window is written every epoch, not just the elements the plan reads: the local
    !! part plus the per-peer runs tile [wFirst,wLast] exactly once, because the old partition
    !! tiles the old element list. So a reused window buffer carries nothing stale from a previous
    !! epoch, and the verification path may compare all of it.
    implicit none
    type(DomainDecomposition),intent(in) :: decomp
    integer,intent(in) :: perElem !! reals per element per variable: (N+1)**2 in 2-D, (N+1)**3 in 3-D
    integer,intent(in) :: nvar
    integer,intent(in) :: nLocalOld !! elements this rank owned before the epoch (uLocal's stride)
    real(prec),intent(in) :: uLocal(*) !! (perElem,nLocalOld,nvar), first element = this rank's first
    integer,intent(in) :: winFirst(1:decomp%nRanks)
    integer,intent(in) :: winLast(1:decomp%nRanks)
    integer,intent(in) :: wFirst !! this rank's window, normalized (wFirst > wLast if empty)
    integer,intent(in) :: wLast
    real(prec),intent(inout) :: uWin(*) !! (perElem,nWinElem,nvar), first element = global old wFirst
    !! intent(inout), not out: the receives write it through MPI rather than through the dummy,
    !! and an intent(out) dummy would license a compiler to treat it as undefined on entry.
    integer,intent(inout) :: requests(:) !! at least 2*nvar*decomp%nRanks entries
    integer,intent(inout) :: msgCount !! 0 on entry; the number of posted messages on exit
    integer(int64),intent(inout) :: nBytesRecv
    integer(int64),intent(inout) :: nBytesSent
    integer(int64),intent(inout) :: nElemRemote
    ! Local
    integer :: r,iv,a,b,cnt,ierror
    integer :: myFirst,nWinElem,nbyte
    ! int64: off spans the whole buffer in reals (perElem*nWinElem*nvar), which a
    ! high-order 3-D window can push past the 32-bit range even though each message's cnt
    ! stays small. Fortran's own subscript arithmetic used to carry this at pointer width;
    ! computing it explicitly is what makes the kind matter.
    integer(int64) :: off

    myFirst = decomp%offsetElem(decomp%rankId+1)+1
    nWinElem = max(wLast-wFirst+1,0)
    ! storage_size, not the kind value prec: the two coincide for the kinds SELF uses, but only
    ! the intrinsic actually reports a width, and these counters are quoted as bytes.
    nbyte = storage_size(1.0_prec)/8

    ! Receives first: the runs of my window that other ranks own.
    do r = 1,decomp%nRanks
      if(r-1 == decomp%rankId) cycle
      call OwnedRun(decomp%offsetElem,r,wFirst,wLast,a,b)
      if(b < a) cycle
      cnt = perElem*(b-a+1)
      nBytesRecv = nBytesRecv+int(cnt,int64)*nvar*nbyte
      nElemRemote = nElemRemote+int(b-a+1,int64)
      do iv = 1,nvar
        off = int(perElem,int64)*(int(a-wFirst,int64)+int(nWinElem,int64)*(iv-1))
        msgCount = msgCount+1
        call MPI_IRECV(uWin(off+1),cnt,decomp%mpiPrec, &
                       r-1,iv,decomp%mpiComm,requests(msgCount),ierror)
      enddo
    enddo

    ! Sends: the runs of my old range that other ranks' windows need. A rank that owns no new
    ! elements still reaches this loop, because its peers may need old elements it owns.
    do r = 1,decomp%nRanks
      if(r-1 == decomp%rankId) cycle
      call OwnedRun(decomp%offsetElem,decomp%rankId+1,winFirst(r),winLast(r),a,b)
      if(b < a) cycle
      cnt = perElem*(b-a+1)
      nBytesSent = nBytesSent+int(cnt,int64)*nvar*nbyte
      do iv = 1,nvar
        off = int(perElem,int64)*(int(a-myFirst,int64)+int(nLocalOld,int64)*(iv-1))
        msgCount = msgCount+1
        call MPI_ISEND(uLocal(off+1),cnt,decomp%mpiPrec, &
                       r-1,iv,decomp%mpiComm,requests(msgCount),ierror)
      enddo
    enddo

  endsubroutine PostOldWindowExchange