Init_DomainDecomposition Subroutine

  1. SELF_DomainDecomposition.f90
  2. SELF_DomainDecomposition
  3. Init_DomainDecomposition

public subroutine Init_DomainDecomposition(this, comm)

Arguments

TypeIntentOptionalAttributesName
class(DomainDecomposition), intent(inout) :: this
integer, intent(in), optional :: comm

Calls

proc~~init_domaindecomposition~~CallsGraph proc~init_domaindecomposition Init_DomainDecomposition proc~acquirempi AcquireMPI proc~init_domaindecomposition->proc~acquirempi hipgetdevicecount hipgetdevicecount proc~init_domaindecomposition->hipgetdevicecount mpi_comm_size mpi_comm_size proc~init_domaindecomposition->mpi_comm_size mpi_comm_rank mpi_comm_rank proc~init_domaindecomposition->mpi_comm_rank mpi_abort mpi_abort proc~init_domaindecomposition->mpi_abort mpi_comm_split_type mpi_comm_split_type proc~init_domaindecomposition->mpi_comm_split_type mpi_comm_free mpi_comm_free proc~init_domaindecomposition->mpi_comm_free hipsetdevice hipsetdevice proc~init_domaindecomposition->hipsetdevice mpi_initialized mpi_initialized proc~acquirempi->mpi_initialized mpi_init mpi_init proc~acquirempi->mpi_init

Contents

Source Code

Init_DomainDecomposition

Source Code

  subroutine Init_DomainDecomposition(this,comm)
    implicit none
    class(DomainDecomposition),intent(inout) :: this
    integer,intent(in),optional :: comm
    ! Local
    integer       :: ierror
    integer       :: nodeComm,localRank
    integer(c_int) :: num_devices,hip_err,device_id

    this%mpiComm = MPI_COMM_NULL
    this%mpiPrec = prec
    this%rankId = 0
    this%nRanks = 1
    this%nElem = 0
    this%mpiEnabled = .false.
    this%ownsMpi = .false.

    call AcquireMPI(this%mpiComm,this%ownsMpi,comm)

    call mpi_comm_rank(this%mpiComm,this%rankId,ierror)
    call mpi_comm_size(this%mpiComm,this%nRanks,ierror)
    print*,__FILE__," : Rank ",this%rankId+1,"/",this%nRanks," checking in."

    if(this%nRanks > 1) then
      this%mpiEnabled = .true.
    else
      print*,__FILE__," : No domain decomposition used."
    endif

    if(prec == real32) then
      this%mpiPrec = MPI_FLOAT
    else
      this%mpiPrec = MPI_DOUBLE
    endif

    allocate(this%offsetElem(1:this%nRanks+1))

    hip_err = hipGetDeviceCount(num_devices)
    if(hip_err /= 0) then
      print*,'Failed to get device count on rank',this%rankId
      call MPI_Abort(this%mpiComm,hip_err,ierror)
    endif

    ! Assign GPU device ID based on node-local MPI rank so multi-node
    ! placement is independent of the launcher's global rank ordering.
    call MPI_Comm_split_type(this%mpiComm,MPI_COMM_TYPE_SHARED,this%rankId, &
                             MPI_INFO_NULL,nodeComm,ierror)
    call MPI_Comm_rank(nodeComm,localRank,ierror)
    call MPI_Comm_free(nodeComm,ierror)
    device_id = modulo(localRank,num_devices)
    hip_err = hipSetDevice(device_id)
    print*,__FILE__," : Rank ",this%rankId+1," assigned to device ",device_id
    if(hip_err /= 0) then
      print*,'Failed to set device for rank',this%rankId,'to device',device_id
      call MPI_Abort(this%mpiComm,hip_err,ierror)
    endif

    this%initialized = .true.

  endsubroutine Init_DomainDecomposition