ghostmodule.f90

来自「CCSM Research Tools: Community Atmospher」· F90 代码 · 共 805 行 · 第 1/2 页

!------------------------------------------------------------------------
!         NASA/GSFC, Data Assimilation Office, Code 910.3, GEOS/DAS
!------------------------------------------------------------------------
      MODULE ghostmodule
!BOP
!
! !MODULE: ghostmodule
!
! !USES:
      USE decompmodule, ONLY : DecompType
#include "debug.h"
#include "pilgrim.h"
      IMPLICIT NONE

!
! !DESCRIPTION:
!
!      This module provides the basic support for "ghost regions".  In
!      reality the ghost region just subset of the global domain 
!      described by a decomposition (pro memoria: a decomposition 
!      describes a partition of a global index space over a number 
!      of PEs; this is inherently non-overlapping).  
!
!      It contains the following public types and routines.
!      \begin{center}
!      \begin{tabular}{|l|l|} \hline \hline
!         GhostType   & Type to describe ghosted local vector \\ \hline
!         GhostFree   & Destroy a ghost definition \\ \hline
!         GhostCreate & Copy ghost definition to newly created one\\ \hline 
!         GhostInfo   & Returns some information about the region \\ \hline 
!         \hline  \hline
!      \end{tabular}
!      \end{center}
!
!      GhostCreate is overloaded to support different types of domains:
!
!      \begin{center}
!      \begin{tabular}{|l|l|} \hline \hline
!         GhostRegular1D & Define a subset of a 1D domain \\ \hline
!         GhostRegular2D & Define a subset of a 2D domain \\ \hline
!         GhostRegular3D & Define a subset of a 3D domain \\ \hline
!         GhostIrregular & Define a subset of an irregular domain \\ \hline
!         \hline  \hline
!      \end{tabular}
!      \end{center}
!
!      Generally one will use the GhostCreate routine which corresponds
!      to the underlying decomposition; e.g., if the decomposition was
!      defined with DecompRegular3D you would probably use GhostRegular3D
!      to define the ghost region.  But since decompositions and ghost
!      regions are generic, i.e., one-size-fits-all, this is not a requirement.
!      Be very careful if you use non-corresponding routines!
!
!      The ghost type contains a decomposition which describes the
!      {\it non-overlapping} distribution of the global domain 
!      (this is a replicated data structure with complete information
!      about all data structures on all PEs).  Its other components are
!      a list of the global indices of the on the boundary 
!      (not replicated), and a description of the mapping of the ghosted
!      local region to global indices.
!
!      This module is communication-free and is a foundation
!      for ParUtilitiesModule.  Since GhostType is local to the
!      PE, the modules routines can and should be called with 
!      non-replicated data structures.  Before boundary communication
!      takes place, the communication pattern derived from the ghost regions
!      must be created (see ParUtilitiesModule).  
!       
! !REVISION HISTORY:
!   00.11.10   Sawyer     Creation
!   01.02.07   Sawyer     Improvements; added Border to GhostType
!   01.02.12   Sawyer     Converted to free format
!
! !PUBLIC TYPES:
      PUBLIC GhostType
      PUBLIC GhostFree
      PUBLIC GhostCreate
      PUBLIC GhostInfo

      INTERFACE     GhostCreate
        MODULE PROCEDURE GhostIrregular
        MODULE PROCEDURE GhostRegular1D
        MODULE PROCEDURE GhostRegular2D
        MODULE PROCEDURE GhostRegular3D
      END INTERFACE
 
! Decomposition info

       TYPE GhostType
         TYPE(DecompType) :: Decomp ! Decomposition of global partition
         TYPE(DecompType) :: Local  ! Decomposition of local region
         TYPE(DecompType) :: Border ! Decomposition of local segment
       END TYPE GhostType

!EOP
      CONTAINS

!-----------------------------------------------------------------------
!BOP
! !IROUTINE: GhostFree --- Free a ghosted region
!
! !INTERFACE:
      SUBROUTINE GhostFree ( Ghost )
! !USES:
      USE decompmodule, ONLY : DecompFree
      IMPLICIT NONE

! !INPUT/OUTPUT PARAMETERS:
      TYPE(GhostType), INTENT( INOUT ):: Ghost  ! Ghost information

!
! !DESCRIPTION:
!     Free the ghost decomposition -- deallocate the data structures.
!
! !SYSTEM ROUTINES:
!     ASSOCIATED, DEALLOCATE
!
! !REVISION HISTORY:
!   00.11.12   Sawyer     Creation
!
!EOP
!-----------------------------------------------------------------------
!BOC
!
!
      CPP_ENTER_PROCEDURE( "GHOSTFREE" )

      CALL DecompFree( Ghost%Border )
      CALL DecompFree( Ghost%Local  )
      CALL DecompFree( Ghost%Decomp ) 

      CPP_LEAVE_PROCEDURE( "GHOSTFREE" )
      RETURN
!EOC
      END SUBROUTINE GhostFree
!-----------------------------------------------------------------------


!-----------------------------------------------------------------------
!BOP
! !IROUTINE: GhostCopy --- Copy one decomposition to another
!
! !INTERFACE:
      SUBROUTINE GhostCopy ( GhostIn, GhostOut )
! !USES:
      USE decompmodule, ONLY : DecompCopy
      IMPLICIT NONE
!
! !INPUT PARAMETERS:
      TYPE(GhostType), INTENT( IN )   :: GhostIn  ! Ghost information
!
! !OUTPUT PARAMETERS:
      TYPE(GhostType), INTENT( OUT )  :: GhostOut ! Ghost information
!
! !DESCRIPTION:
!
!   Creates an output ghost definition and copies GhostIn to it 
!
! !SYSTEM ROUTINES:
!     ALLOCATE
!
! !REVISION HISTORY:
!   00.11.12   Sawyer     Creation
!
!EOP
!-----------------------------------------------------------------------
!BOC
! !LOCAL VARIABLES:
      INTEGER  :: I, Nsize

      CPP_ENTER_PROCEDURE( "GHOSTCOPY" )

      CALL DecompCopy( GhostIn%Decomp, GhostOut%Decomp )
      CALL DecompCopy( GhostIn%Local,  GhostOut%Local  )
      CALL DecompCopy( GhostIn%Border, GhostOut%Border )

      CPP_LEAVE_PROCEDURE( "GHOSTCOPY" )
      RETURN
!EOC
      END SUBROUTINE GhostCopy
!-----------------------------------------------------------------------


!-----------------------------------------------------------------------
!BOP
! !IROUTINE: GhostIrregular --- Create a ghost definition for 1-D grid
!
! !INTERFACE:
      SUBROUTINE GhostIrregular( Decomp, Id, LocalSize, Tags, Ghost )
! !USES:
      USE decompmodule, ONLY : DecompCreate, DecompCopy,                 &
     &                         DecompGlobalToLocal, DecompInfo
      IMPLICIT NONE
!
! !INPUT PARAMETERS:
      TYPE(DecompType), INTENT( IN ) :: Decomp     ! Decomp information
      INTEGER, INTENT( IN )          :: Id         ! Local PE identifer
      INTEGER, INTENT( IN )          :: LocalSize  ! Size of local segment
      INTEGER, INTENT( IN )          :: Tags(:)    ! Global tags
!
! !OUTPUT PARAMETERS:
      TYPE(GhostType), INTENT( OUT ) :: Ghost  ! Ghost definition
!
!
! !DESCRIPTION:
!     Creates a ghost definition for a ghosted array given by
!     the PEs and Tags of the local points.  Note that none of the 
!     array bounds can be outside the global domain!
!
! !SYSTEM ROUTINES:
!     ALLOCATE, DEALLOCATE
!
! !REVISION HISTORY:
!   00.11.12   Sawyer     Creation
!
! !BUGS:
!   None of the array bounds can be outside of the global domain!
!   This is significant if the local region is on the edge of the
!   domain, and, in other words, the ghost region cannot cover
!   empty space.  This limitation may be relaxed in the future.
!
!EOP
!-----------------------------------------------------------------------
!BOC
! !LOCAL VARIABLES:
      INTEGER :: I, NPEs, GlobalSize, Local, Count, Ipe
      INTEGER, ALLOCATABLE :: Pe(:), Other(:)
!
!
      CPP_ENTER_PROCEDURE( "GHOSTIRREGULAR" )
!
! Allocate the basic data structures
!
      CALL DecompInfo( Decomp, Npes, GlobalSize )

      ALLOCATE( Pe( LocalSize ) )
      ALLOCATE( Other( LocalSize ) )

!
! Use decompmodule to create global and local portions of Ghost
! The local version is only on the local processor "0"
 
      Other = Id
      CALL DecompCreate( Npes, Other, LocalSize, Tags, Ghost%Local )

!
! Perform over all points local segment
!
      Count = 0
      DO I= 1, LocalSize
        CALL DecompGlobalToLocal( Decomp, Tags(I), Local, Ipe )
        CPP_ASSERT_F90( (Local .GT. 0) .AND. (ipe .GE. 0) )
        IF ( Ipe .ne. id ) THEN
          Count = Count + 1
          Other( Count ) = Tags(I)
          Pe( Count )    = Ipe
        ENDIF
      ENDDO

!
! Define the border regions.  Presumably Count << LocalSize
!
      CALL DecompCreate( Npes, Pe, Count, Other, Ghost%Border )

!
! Copy the decomposition too
!
      CALL DecompCopy( Decomp, Ghost%Decomp )

! Clean up

      DEALLOCATE( Pe )
      DEALLOCATE( Other )

      CPP_LEAVE_PROCEDURE( "GHOSTIRREGULAR" )
      RETURN
!EOC
      END SUBROUTINE GhostIrregular
!-----------------------------------------------------------------------


!-----------------------------------------------------------------------
!BOP
! !IROUTINE: GhostRegular1D --- Create a ghost definition for 1-D grid
!
! !INTERFACE:
      SUBROUTINE GhostRegular1D( Decomp, Id, Xglobal, Xfrom, Xto, Xwrap, &
     &                           Ghost )
! !USES:
      USE decompmodule, ONLY : DecompCreate, DecompCopy,                 &
     &                         DecompGlobalToLocal, DecompInfo
      IMPLICIT NONE
!
! !INPUT PARAMETERS:
      TYPE(DecompType), INTENT( IN )   :: Decomp ! Decomp information
      INTEGER, INTENT( IN )            :: Id     ! Local PE identifer
      INTEGER, INTENT( IN )            :: Xglobal! Total in X
      INTEGER, INTENT( IN )            :: Xfrom  ! Low index in X
      INTEGER, INTENT( IN )            :: Xto    ! High index in X
      LOGICAL, INTENT( IN )            :: Xwrap  ! Wrap in X?
!
! !OUTPUT PARAMETERS:p
      TYPE(GhostType), INTENT( OUT )   :: Ghost  ! Ghost definition
!
!
! !DESCRIPTION:
!     Creates a ghost definition for a regular 1-D array with the
!     array bounds Xfrom:Xto.
!
!     If the array bounds are outside of the global domain they may
!     be wrapped around back into the global domain (variable Xwrap).  
!     If the region is not wrapped, it is advisable that the ghost 
!     region end at the boundary (which usually requires
!     special case treatment depending on the PE number). If 
!     it does not end at the boundary, undefined points are 
!     introduced.
!
! !SYSTEM ROUTINES:
!     ALLOCATE, DEALLOCATE
!
! !REVISION HISTORY:
!   00.11.12   Sawyer     Creation
!
! !BUGS:
!
!   There are certain limitations to ghost regions which can be
!   avoided by clean programming practices.  If the ghosted region
!   wraps back onto core regions of the same PE, problems can arise.  
!   The simple case -- a ghosted region on 1 PE -- is supported in
!   most cases.  However, if it wraps back onto the local PE 
!   in such a way that more than one ghost points is mapped to
!   one core domain global index, then the code may fail.  Note
!   that this is rarely the case if the ghost regions are small
!   and enough processors are used to avoid wrapping back on the
!   local one.
!   
!EOP
!-----------------------------------------------------------------------
!BOC
! !LOCAL VARIABLES:
      INTEGER :: I, L, NPEs, GlobalSize, LocalSize, Count, Local, Ipe
      INTEGER :: Global
      INTEGER, ALLOCATABLE :: Pe(:), Tags(:), Other(:)
!
!
      CPP_ENTER_PROCEDURE( "GHOSTREGULAR1D" )

!
! Allocate the basic data structures
!
      CALL DecompInfo( Decomp, NPEs, GlobalSize )
      CPP_ASSERT_F90( GlobalSize .EQ. Xglobal )

      LocalSize = Xto - Xfrom + 1
      CPP_ASSERT_F90( LocalSize .GE. 0 )

      ALLOCATE( Pe( LocalSize ) )
      ALLOCATE( Tags( LocalSize ) )
      ALLOCATE( Other( LocalSize ) )

!
! Perform over all points local segment
!
      Count = 0
      L = 0
      DO I = Xfrom, Xto
        L = L + 1
        Global = MODULO(I-1,Xglobal)+1  ! Wrap around condition
        IF (Xwrap .OR. Global==I) THEN
          Tags(L) = Global                 ! Global Tags
          CALL DecompGlobalToLocal( Decomp, Global, Local, Ipe )
          IF ( Ipe .ne. Id .AND. Ipe .GE. 0 ) THEN
            Count = Count + 1
            Other( Count ) = Global        ! Local Tags
            Pe( Count )    = Ipe
          ENDIF
!
! Special case: the domain wraps-around onto the same PE.  This is
! very tricky:  the ghost points are distinguished from their true
! local core domain counterparts by a minus sign.  This makes the
! address space in both Ghost%Border and Ghost%Local unique
!
          IF ( Ipe .eq. Id .AND. I .ne. Global ) THEN
            Count = Count + 1
            Other( Count ) = -Global       ! Local Tags
            Pe( Count )    = Ipe
            Tags(L) = -Global              ! Global Tags (mark ghost region!)
          ENDIF
        ELSE
          Tags(L) = 0
        ENDIF
      ENDDO

!
! Perform over all points local segment
!
      CALL DecompCreate( Npes, Pe, Count, Other, Ghost%Border )

!
! Use decompmodule to create global and local portions of Ghost
! The local version is only on the local PE
!
      Other = Id