Skip to content

Optimize ACC port of atm_advance_scalars_mono_work:5555 #1494

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
abishekg7 wants to merge 5 commits into
base: develop
Choose a base branch
from
+21 −18
Draft

Optimize ACC port of atm_advance_scalars_mono_work:5555 #1494

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
00a610b
Add timers
abishekg7 Jul 23, 2026
0c4e7d0
Opt 1 - Split up into different parallel regions
abishekg7 Jul 23, 2026
56d4583
Opt 2 - Use vector collapse(2) for last two parallel regions
abishekg7 Jul 23, 2026
5dba280
Opt 3 - Swap loop vector and seq in first parallel region
abishekg7 Jul 23, 2026
937ff3a
Opt 4 - Remove vector collapse(2) in 2nd & 3rd parallel regions
abishekg7 Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 21 additions & 18 deletions src/core_atmosphere/dynamics/mpas_atm_time_integration.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5165,6 +5165,7 @@ subroutine atm_advance_scalars_mono_work(field_name, block, state, nCells, nEdge
real (kind=RKIND), dimension(nVertLevels) :: flux_upwind_arr
real (kind=RKIND) :: flux3, flux4, flux_upwind
real (kind=RKIND) :: q_im2, q_im1, q_i, q_ip1, ua, coef3
real (kind=RKIND) :: iac, eoc_s, f_t
#ifdef DEBUG_TRANSPORT
real (kind=RKIND) :: scmin,scmax
#endif
Expand Down Expand Up @@ -5551,29 +5552,28 @@ subroutine atm_advance_scalars_mono_work(field_name, block, state, nCells, nEdge
!$acc end parallel

!$OMP BARRIER

call mpas_timer_start('atm_advance_scalars_mono_work_5555')
!$acc parallel

!$acc loop gang worker
do iCell=cellSolveStart,cellSolveEnd

!$acc loop seq
do i=1,nEdgesOnCell(iCell)
iEdge = edgesOnCell(i,iCell)

!$acc loop vector
do k=1, nVertLevels
scalar_new(k,iCell) = scalar_new(k,iCell) - edgesOnCell_sign(i,iCell) * flux_upwind_tmp(k,iEdge) * invAreaCell(iCell)

scale_arr(k,SCALE_OUT,iCell) = scale_arr(k,SCALE_OUT,iCell) &
- max(0.0_RKIND,edgesOnCell_sign(i,iCell)*flux_tmp(k,iEdge)) * invAreaCell(iCell)
scale_arr(k,SCALE_IN, iCell) = scale_arr(k,SCALE_IN, iCell) &
- min(0.0_RKIND,edgesOnCell_sign(i,iCell)*flux_tmp(k,iEdge)) * invAreaCell(iCell)
iac = invAreaCell(iCell)
!$acc loop vector
do k=1, nVertLevels
!$acc loop seq
do i=1,nEdgesOnCell(iCell)
iEdge = edgesOnCell(i,iCell)
eoc_s = edgesOnCell_sign(i,iCell)
f_t = flux_tmp(k,iEdge)

scalar_new(k,iCell) = scalar_new(k,iCell) - eoc_s * flux_upwind_tmp(k,iEdge) * iac
scale_arr(k,SCALE_OUT,iCell) = scale_arr(k,SCALE_OUT,iCell) - max(0.0_RKIND,eoc_s*f_t) * iac
scale_arr(k,SCALE_IN, iCell) = scale_arr(k,SCALE_IN, iCell) - min(0.0_RKIND,eoc_s*f_t) * iac
end do

end do
end do

!$acc end parallel

!
! next, the limiter
Expand All @@ -5583,9 +5583,9 @@ subroutine atm_advance_scalars_mono_work(field_name, block, state, nCells, nEdge
! worked through algebra and found equivalent form
! added benefit that it should address ifort single prec overflow issue
if (local_advance_density) then
!$acc parallel
!$acc loop gang worker
do iCell=cellSolveStart,cellSolveEnd

!$acc loop vector
do k = 1, nVertLevels
scale_factor = (s_max(k,iCell)*rho_zz_int(k,iCell) - scalar_new(k,iCell)) / &
Expand All @@ -5597,10 +5597,11 @@ subroutine atm_advance_scalars_mono_work(field_name, block, state, nCells, nEdge
scale_arr(k,SCALE_OUT,iCell) = min( 1.0_RKIND, max( 0.0_RKIND, scale_factor) )
end do
end do
!$acc end parallel
else
!$acc parallel
!$acc loop gang worker
do iCell=cellSolveStart,cellSolveEnd

!$acc loop vector
do k = 1, nVertLevels
scale_factor = (s_max(k,iCell)*rho_zz_new(k,iCell) - scalar_new(k,iCell)) / &
Expand All @@ -5612,9 +5613,11 @@ subroutine atm_advance_scalars_mono_work(field_name, block, state, nCells, nEdge
scale_arr(k,SCALE_OUT,iCell) = min( 1.0_RKIND, max( 0.0_RKIND, scale_factor) )
end do
end do
!$acc end parallel
end if

!$acc end parallel

call mpas_timer_stop('atm_advance_scalars_mono_work_5555')

!
! communicate scale factors here.
Expand Down