cheby_device.F90

Go to the documentation of this file.

! Copyright (c) 2024-2025, The Neko Authors
! All rights reserved.
!
! Redistribution and use in source and binary forms, with or without
! modification, are permitted provided that the following conditions
! are met:
!
!   * Redistributions of source code must retain the above copyright
!     notice, this list of conditions and the following disclaimer.
!
!   * Redistributions in binary form must reproduce the above
!     copyright notice, this list of conditions and the following
!     disclaimer in the documentation and/or other materials provided
!     with the distribution.
!
!   * Neither the name of the authors nor the names of its
!     contributors may be used to endorse or promote products derived
!     from this software without specific prior written permission.
!
! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
! "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
! LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
! FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
! COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
! INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
! BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
! LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
! CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
! LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
! ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
! POSSIBILITY OF SUCH DAMAGE.
!
module cheby_device
  use krylov, only : ksp_t, ksp_monitor_t
  use precon, only : pc_t
  use ax_product, only : ax_t
  use num_types, only : rp, c_rp
  use profiler, only : profiler_start_region, profiler_end_region
  use field, only : field_t
  use coefs, only : coef_t
  use mesh, only : mesh_t
  use space, only : space_t
  use gather_scatter, only : gs_t, gs_op_add
  use scalar_bc_projector, only : scalar_bc_projector_t
  use vector_bc_projector, only : vector_bc_projector_t, &
       vector_bc_projector_components
  use schwarz, only : schwarz_t
  use device_math, only : device_cmult2, device_sub2, &
       device_add2s1, device_add2s2, device_glsc3, device_copy, &
       device_sub3, device_cmult, device_add2
  use device, only : glb_cmd_queue, device_map, device_event_create, &
       device_unmap, device_event_destroy, device_memcpy, host_to_device, &
       device_get_ptr
  use, intrinsic :: iso_c_binding, only : c_ptr, c_int, &
       c_null_ptr, c_associated
  implicit none
  private
 
  type, public, extends(ksp_t) :: cheby_device_t
     real(kind=rp), allocatable :: d(:)
     real(kind=rp), allocatable :: w(:)
     real(kind=rp), allocatable :: r(:)
     type(c_ptr) :: d_d = c_null_ptr
     type(c_ptr) :: w_d = c_null_ptr
     type(c_ptr) :: r_d = c_null_ptr
     type(c_ptr) :: gs_event = c_null_ptr
     real(kind=rp) :: tha, dlt
     integer :: power_its = 150
     integer :: power_its_refresh = 20
     logical :: warm_start_eigs = .false.
     logical :: eigs_computed = .false.
     real(kind=rp), allocatable :: ev(:)
     type(c_ptr) :: ev_d = c_null_ptr
     logical :: recompute_eigs = .true.
     logical :: zero_initial_guess = .false.
     type(schwarz_t), pointer :: schwarz => null() 
   contains
     procedure, pass(this) :: init => cheby_device_init
     procedure, pass(this) :: free => cheby_device_free
     procedure, pass(this) :: solve => cheby_device_impl
     procedure, pass(this) :: solve_coupled => cheby_device_solve_coupled
  end type cheby_device_t
 
#ifdef HAVE_HIP
  interface
     subroutine hip_cheby_device_part1(d_d, x_d, inv_tha, n, strm) &
          bind(c, name = 'hip_cheby_part1')
       use, intrinsic :: iso_c_binding
       import c_rp
       implicit none
       type(c_ptr), value :: d_d, x_d, strm
       real(c_rp) :: inv_tha
       integer(c_int) :: n
     end subroutine hip_cheby_device_part1
  end interface
 
  interface
     subroutine hip_cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n, strm) &
          bind(c, name = 'hip_cheby_part2')
       use, intrinsic :: iso_c_binding
       import c_rp
       implicit none
       type(c_ptr), value :: d_d, w_d, x_d, strm
       real(c_rp) :: tmp1, tmp2
       integer(c_int) :: n
     end subroutine hip_cheby_device_part2
  end interface
#elif HAVE_CUDA
  interface
     subroutine cuda_cheby_device_part1(d_d, x_d, inv_tha, n, strm) &
          bind(c, name = 'cuda_cheby_part1')
       use, intrinsic :: iso_c_binding
       import c_rp
       implicit none
       type(c_ptr), value :: d_d, x_d, strm
       real(c_rp) :: inv_tha
       integer(c_int) :: n
     end subroutine cuda_cheby_device_part1
  end interface
 
  interface
     subroutine cuda_cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n, strm) &
          bind(c, name = 'cuda_cheby_part2')
       use, intrinsic :: iso_c_binding
       import c_rp
       implicit none
       type(c_ptr), value :: d_d, w_d, x_d, strm
       real(c_rp) :: tmp1, tmp2
       integer(c_int) :: n
     end subroutine cuda_cheby_device_part2
  end interface
#elif HAVE_METAL
  interface
     subroutine metal_cheby_device_part1(d_d, x_d, inv_tha, n, strm) &
          bind(c, name = 'metal_cheby_part1')
       use, intrinsic :: iso_c_binding
       import c_rp
       implicit none
       type(c_ptr), value :: d_d, x_d, strm
       real(c_rp) :: inv_tha
       integer(c_int) :: n
     end subroutine metal_cheby_device_part1
  end interface
 
  interface
     subroutine metal_cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n, strm) &
          bind(c, name = 'metal_cheby_part2')
       use, intrinsic :: iso_c_binding
       import c_rp
       implicit none
       type(c_ptr), value :: d_d, w_d, x_d, strm
       real(c_rp) :: tmp1, tmp2
       integer(c_int) :: n
     end subroutine metal_cheby_device_part2
  end interface
#endif
 
contains
  subroutine cheby_device_part1(d_d, x_d, inv_tha, n)
    type(c_ptr) :: d_d, x_d
    real(c_rp) :: inv_tha
    integer(c_int) :: n
#ifdef HAVE_HIP
    call hip_cheby_device_part1(d_d, x_d, inv_tha, n, glb_cmd_queue)
#elif HAVE_CUDA
    call cuda_cheby_device_part1(d_d, x_d, inv_tha, n, glb_cmd_queue)
#elif HAVE_METAL
    call metal_cheby_device_part1(d_d, x_d, inv_tha, n, glb_cmd_queue)
#else !Fallback to device_math for missing device kernels
    call device_cmult( d_d, inv_tha, n)
    call device_add2( x_d, d_d, n)
#endif
  end subroutine cheby_device_part1
 
 
 
  subroutine cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n)
    type(c_ptr) :: d_d, w_d, x_d
    real(c_rp) :: tmp1, tmp2
    integer(c_int) :: n
#ifdef HAVE_HIP
    call hip_cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n, glb_cmd_queue)
#elif HAVE_CUDA
    call cuda_cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n, glb_cmd_queue)
#elif HAVE_METAL
    call metal_cheby_device_part2(d_d, w_d, x_d, tmp1, tmp2, n, glb_cmd_queue)
#else !Fallback to device_math for missing device kernels
    call device_cmult( d_d, tmp1, n)
    call device_add2s2( d_d, w_d, tmp2, n)
    call device_add2( x_d, d_d, n)
#endif
  end subroutine cheby_device_part2
 
  subroutine cheby_device_init(this, n, max_iter, M, rel_tol, abs_tol, monitor)
    class(cheby_device_t), intent(inout), target :: this
    integer, intent(in) :: max_iter
    class(pc_t), optional, intent(in), target :: M
    integer, intent(in) :: n
    real(kind=rp), optional, intent(in) :: rel_tol
    real(kind=rp), optional, intent(in) :: abs_tol
    logical, optional, intent(in) :: monitor
 
    call this%free()
    allocate(this%d(n))
    allocate(this%w(n))
    allocate(this%r(n))
 
    call device_map(this%d, this%d_d, n)
    call device_map(this%w, this%w_d, n)
    call device_map(this%r, this%r_d, n)
 
    if (present(m)) then
       this%M => m
    end if
 
    if (present(rel_tol) .and. present(abs_tol) .and. present(monitor)) then
       call this%ksp_init(max_iter, rel_tol, abs_tol, monitor = monitor)
    else if (present(rel_tol) .and. present(abs_tol)) then
       call this%ksp_init(max_iter, rel_tol, abs_tol)
    else if (present(monitor) .and. present(abs_tol)) then
       call this%ksp_init(max_iter, abs_tol = abs_tol, monitor = monitor)
    else if (present(rel_tol) .and. present(monitor)) then
       call this%ksp_init(max_iter, rel_tol, monitor = monitor)
    else if (present(rel_tol)) then
       call this%ksp_init(max_iter, rel_tol = rel_tol)
    else if (present(abs_tol)) then
       call this%ksp_init(max_iter, abs_tol = abs_tol)
    else if (present(monitor)) then
       call this%ksp_init(max_iter, monitor = monitor)
    else
       call this%ksp_init(max_iter)
    end if
 
    call device_event_create(this%gs_event, 2)
 
  end subroutine cheby_device_init
 
  subroutine cheby_device_free(this)
    class(cheby_device_t), intent(inout) :: this
 
    call this%ksp_free()
 
    if (allocated(this%d)) then
       if (c_associated(this%d_d)) then
          call device_unmap(this%d, this%d_d)
       end if
       deallocate(this%d)
    end if
 
    if (allocated(this%w)) then
       if (c_associated(this%w_d)) then
          call device_unmap(this%w, this%w_d)
       end if
       deallocate(this%w)
    end if
 
    if (allocated(this%r)) then
       if (c_associated(this%r_d)) then
          call device_unmap(this%r, this%r_d)
       end if
       deallocate(this%r)
    end if
 
    if (allocated(this%ev)) then
       if (c_associated(this%ev_d)) then
          call device_unmap(this%ev, this%ev_d)
       end if
       deallocate(this%ev)
    end if
 
    nullify(this%M)
 
    if (c_associated(this%gs_event)) then
       call device_event_destroy(this%gs_event)
    end if
 
  end subroutine cheby_device_free
 
  subroutine cheby_device_power(this, Ax, x, n, coef, bc_projector, gs_h)
    class(cheby_device_t), intent(inout) :: this
    class(ax_t), intent(in) :: Ax
    type(field_t), intent(inout) :: x
    integer, intent(in) :: n
    type(coef_t), intent(inout) :: coef
    class(scalar_bc_projector_t), intent(inout) :: bc_projector
    type(gs_t), intent(inout) :: gs_h
    real(kind=rp) :: lam, b, a, rn
    real(kind=rp) :: boost = 1.1_rp
    real(kind=rp) :: lam_factor = 30.0_rp
    real(kind=rp) :: wtw, dtw, dtd
    integer, allocatable :: fixed_seed(:), saved_seed(:)
    integer :: i, rnd_n, its
    logical :: warm
 
    call profiler_start_region('cheby_power')
    warm = this%warm_start_eigs .and. this%eigs_computed .and. &
         c_associated(this%ev_d)
    associate(w => this%w, w_d => this%w_d, d => this%d, d_d => this%d_d)
 
      if (warm) then
         its = this%power_its_refresh
         call device_copy(d_d, this%ev_d, n)
      else
         its = this%power_its
 
         ! Save current random seed and set a fixed seed
         call random_seed(size = rnd_n)
         allocate(saved_seed(rnd_n))
         allocate(fixed_seed(rnd_n))
         fixed_seed = 3901
         call random_seed(get = saved_seed)
         call random_seed(put = fixed_seed)
 
         do i = 1, n
            call random_number(rn)
            d(i) = rn + 10.0_rp
         end do
         call device_memcpy(d, d_d, n, host_to_device, sync = .true.)
 
         ! Restore saved random seed
         call random_seed(put = saved_seed)
 
         call gs_h%op(d, n, gs_op_add, this%gs_event)
         call bc_projector%apply(d, n)
      end if
 
      !Power method to get lambda max
      do i = 1, its
         call ax%compute(w, d, coef, x%msh, x%Xh)
         call gs_h%op(w, n, gs_op_add, this%gs_event)
         call bc_projector%apply(w, n)
         if (associated(this%schwarz)) then
            call this%schwarz%compute(this%r, w)
            call device_copy(w_d, this%r_d, n)
         else
            call this%M%solve(this%r, w, n)
            call device_copy(w_d, this%r_d, n)
         end if
 
         wtw = device_glsc3(w_d, coef%mult_d, w_d, n)
         call device_cmult2(d_d, w_d, 1.0_rp/sqrt(wtw), n)
         call bc_projector%apply(d, n)
      end do
 
      call ax%compute(w, d, coef, x%msh, x%Xh)
      call gs_h%op(w, n, gs_op_add, this%gs_event)
      call bc_projector%apply(w, n)
      if (associated(this%schwarz)) then
         call this%schwarz%compute(this%r, w)
         call device_copy(w_d, this%r_d, n)
      else
         call this%M%solve(this%r, w, n)
         call device_copy(w_d, this%r_d, n)
      end if
 
      dtw = device_glsc3(d_d, coef%mult_d, w_d, n)
      dtd = device_glsc3(d_d, coef%mult_d, d_d, n)
      lam = dtw / dtd
      b = lam * boost
      a = lam / lam_factor
      this%tha = (b+a)/2.0_rp
      this%dlt = (b-a)/2.0_rp
 
      if (this%warm_start_eigs) then
         if (.not. c_associated(this%ev_d)) then
            allocate(this%ev(size(this%d)))
            call device_map(this%ev, this%ev_d, size(this%d))
         end if
         call device_copy(this%ev_d, d_d, n)
      end if
      this%eigs_computed = .true.
 
      this%recompute_eigs = .false.
    end associate
    call profiler_end_region('cheby_power')
  end subroutine cheby_device_power
 
  function cheby_device_solve(this, Ax, x, f, n, coef, bc_projector, gs_h, &
       niter) result(ksp_results)
    class(cheby_device_t), intent(inout) :: this
    class(ax_t), intent(in) :: ax
    type(field_t), intent(inout) :: x
    integer, intent(in) :: n
    real(kind=rp), dimension(n), intent(in) :: f
    type(coef_t), intent(inout) :: coef
    class(scalar_bc_projector_t), intent(inout) :: bc_projector
    type(gs_t), intent(inout) :: gs_h
    type(ksp_monitor_t) :: ksp_results
    integer, optional, intent(in) :: niter
    integer :: iter, max_iter
    real(kind=rp) :: a, b, rtr, rnorm, norm_fac
    type(c_ptr) :: f_d
 
    f_d = device_get_ptr(f)
 
    if (this%recompute_eigs) then
       call cheby_device_power(this, ax, x, n, coef, bc_projector, gs_h)
    end if
 
    if (present(niter)) then
       max_iter = niter
    else
       max_iter = this%max_iter
    end if
    norm_fac = 1.0_rp / sqrt(coef%volume)
 
    associate( w => this%w, r => this%r, d => this%d, &
         w_d => this%w_d, r_d => this%r_d, d_d => this%d_d)
      ! calculate residual
      call device_copy(r_d, f_d, n)
      call ax%compute(w, x%x, coef, x%msh, x%Xh)
      call gs_h%op(w, n, gs_op_add, this%gs_event)
      call bc_projector%apply(w, n)
      call device_sub2(r_d, w_d, n)
 
      rtr = device_glsc3(r_d, coef%mult_d, r_d, n)
      rnorm = sqrt(rtr) * norm_fac
      ksp_results%res_start = rnorm
      ksp_results%res_final = rnorm
      ksp_results%iter = 0
 
      ! First iteration
      call this%M%solve(w, r, n)
      call device_copy(d_d, w_d, n)
      a = 2.0_rp / this%tha
      call device_add2s2(x%x_d, d_d, a, n)! x = x + a*d
 
      ! Rest of the iterations
      do iter = 2, max_iter
         ! calculate residual
         call device_copy(r_d, f_d, n)
         call ax%compute(w, x%x, coef, x%msh, x%Xh)
         call gs_h%op(w, n, gs_op_add, this%gs_event)
         call bc_projector%apply(w, n)
         call device_sub2(r_d, w_d, n)
 
         call this%M%solve(w, r, n)
 
         if (iter .eq. 2) then
            b = 0.5_rp * (this%dlt * a)**2
         else
            b = (this%dlt * a / 2.0_rp)**2
         end if
         a = 1.0_rp/(this%tha - b/a)
         call device_add2s1(d_d, w_d, b, n)! d = w + b*d
 
         call device_add2s2(x%x_d, d_d, a, n)! x = x + a*d
      end do
 
      ! calculate residual
      call device_copy(r_d, f_d, n)
      call ax%compute(w, x%x, coef, x%msh, x%Xh)
      call gs_h%op(w, n, gs_op_add, this%gs_event)
      call bc_projector%apply(w, n)
      call device_sub2(r_d, w_d, n)
      rtr = device_glsc3(r_d, coef%mult_d, r_d, n)
      rnorm = sqrt(rtr) * norm_fac
 
 
      ksp_results%res_final = rnorm
      ksp_results%iter = iter
      ksp_results%converged = this%is_converged(iter, rnorm)
    end associate
  end function cheby_device_solve
 
  function cheby_device_impl(this, Ax, x, f, n, coef, bc_projector, gs_h, &
       niter) result(ksp_results)
    class(cheby_device_t), intent(inout) :: this
    class(ax_t), intent(in) :: ax
    type(field_t), intent(inout) :: x
    integer, intent(in) :: n
    real(kind=rp), dimension(n), intent(in) :: f
    type(coef_t), intent(inout) :: coef
    class(scalar_bc_projector_t), intent(inout) :: bc_projector
    type(gs_t), intent(inout) :: gs_h
    type(ksp_monitor_t) :: ksp_results
    integer, optional, intent(in) :: niter
    integer :: iter, max_iter
    real(kind=rp) :: a, b, rtr, rnorm, norm_fac
    real(kind=rp) :: rhok, rhokp1, sig1, tmp1, tmp2
    type(c_ptr) :: f_d
 
    f_d = device_get_ptr(f)
 
    if (this%recompute_eigs) then
       call cheby_device_power(this, ax, x, n, coef, bc_projector, gs_h)
    end if
 
    if (present(niter)) then
       max_iter = niter
    else
       max_iter = this%max_iter
    end if
    norm_fac = 1.0_rp / sqrt(coef%volume)
 
    associate( w => this%w, r => this%r, d => this%d, &
         w_d => this%w_d, r_d => this%r_d, d_d => this%d_d)
      ! calculate residual
      if (.not.this%zero_initial_guess) then
         call ax%compute(w, x%x, coef, x%msh, x%Xh)
         call gs_h%op(w, n, gs_op_add, this%gs_event)
         call bc_projector%apply(w, n)
         call device_sub3(r_d, f_d, w_d, n)
      else
         call device_copy(r_d, f_d, n)
         this%zero_initial_guess = .false.
      end if
 
      ! First iteration
      if (associated(this%schwarz)) then
         call this%schwarz%compute(d, r)
      else
         call this%M%solve(d, r, n)
      end if
 
      tmp1 = 1.0_rp / this%tha
      call cheby_device_part1(d_d, x%x_d, tmp1, n)
 
      sig1 = this%tha / this%dlt
      rhok = 1.0_rp / sig1
 
      ! Rest of the iterations
      do iter = 2, max_iter
         rhokp1 = 1.0_rp / (2.0_rp * sig1 - rhok)
         tmp1 = rhokp1 * rhok
         tmp2 = 2.0_rp * rhokp1 / this%dlt
         rhok = rhokp1
         ! calculate residual
         call ax%compute(w, x%x, coef, x%msh, x%Xh)
         call gs_h%op(w, n, gs_op_add, this%gs_event)
         call bc_projector%apply(w, n)
         call device_sub3(r_d, f_d, w_d, n)
 
         if (associated(this%schwarz)) then
            call this%schwarz%compute(w, r)
         else
            call this%M%solve(w, r, n)
         end if
 
         call cheby_device_part2(d_d, w_d, x%x_d, tmp1, tmp2, n)
 
      end do
 
    end associate
  end function cheby_device_impl
 
  function cheby_device_solve_coupled(this, Ax, x, y, z, fx, fy, fz, &
       n, coef, bc_projector, gs_h, niter) result(ksp_results)
    class(cheby_device_t), intent(inout) :: this
    class(ax_t), intent(in) :: ax
    type(field_t), intent(inout) :: x
    type(field_t), intent(inout) :: y
    type(field_t), intent(inout) :: z
    integer, intent(in) :: n
    real(kind=rp), dimension(n), intent(in) :: fx
    real(kind=rp), dimension(n), intent(in) :: fy
    real(kind=rp), dimension(n), intent(in) :: fz
    type(coef_t), intent(inout) :: coef
    class(vector_bc_projector_t), intent(inout) :: bc_projector
    type(gs_t), intent(inout) :: gs_h
    type(ksp_monitor_t), dimension(3) :: ksp_results
    integer, optional, intent(in) :: niter
    type(scalar_bc_projector_t), pointer :: bc_x, bc_y, bc_z
 
    call vector_bc_projector_components(bc_projector, bc_x, bc_y, bc_z)
    ksp_results(1) = this%solve(ax, x, fx, n, coef, bc_x, gs_h, niter)
    ksp_results(2) = this%solve(ax, y, fy, n, coef, bc_y, gs_h, niter)
    ksp_results(3) = this%solve(ax, z, fz, n, coef, bc_z, gs_h, niter)
 
  end function cheby_device_solve_coupled
 
end module cheby_device