From d15259e409073c15537f57d32351b92887c684b7 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jun 2026 15:35:11 -0600 Subject: [PATCH 1/6] Add timer adv_scalars_4760 --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 227fbde862..6f15600f45 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4756,7 +4756,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end if weight_time_old = 1. - weight_time_new - + call mpas_timer_start('atm_advance_scalars_4760') !$acc parallel async !$acc loop gang worker private(scalar_weight2, ica) do iEdge=edgeStart,edgeEnd @@ -4842,7 +4842,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end if ! end of regional MPAS test end do !$acc end parallel - + call mpas_timer_stop('atm_advance_scalars_4760') !$OMP BARRIER ! From 20de29662b5c854c6d415b007eae7378cff9aff4 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jun 2026 16:15:43 -0600 Subject: [PATCH 2/6] Opt 1 - Rearrange and fuse loops in case default In case default, swap the seq and vector collapse(2) loops and fuse horiz_flux_arr --- .../dynamics/mpas_atm_time_integration.F | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 6f15600f45..1bc6dcc868 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4803,16 +4803,9 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & do k=1,nVertLevels do iScalar=1,num_scalars horiz_flux_arr(iScalar,k,iEdge) = 0.0_RKIND - end do - end do - - !$acc loop seq - do j=1,nAdvCellsForEdge(iEdge) - iAdvCell = advCellsForEdge(j,iEdge) - - !$acc loop vector collapse(2) - do k=1,nVertLevels - do iScalar=1,num_scalars + !$acc loop seq + do j=1,nAdvCellsForEdge(iEdge) + iAdvCell = advCellsForEdge(j,iEdge) scalar_weight = adv_coefs(j,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(j,iEdge) horiz_flux_arr(iScalar,k,iEdge) = horiz_flux_arr(iScalar,k,iEdge) & + scalar_weight * scalar_new(iScalar,k,iAdvCell) From 0f1b1e6af1774513c63e5634bc7c48de603ed461 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jun 2026 16:50:37 -0600 Subject: [PATCH 3/6] Opt 2 - Remove ica loop, directly subst advCellsForEdge --- .../dynamics/mpas_atm_time_integration.F | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 1bc6dcc868..9fea1c1545 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4758,7 +4758,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & call mpas_timer_start('atm_advance_scalars_4760') !$acc parallel async - !$acc loop gang worker private(scalar_weight2, ica) + !$acc loop gang worker private(scalar_weight2) do iEdge=edgeStart,edgeEnd if ((.not.config_apply_lbcs) & @@ -4775,25 +4775,20 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end do end do - !$acc loop vector - do j=1,10 - ica(j) = advCellsForEdge(j,iEdge) - end do - !$acc loop vector collapse(2) do k = 1,nVertLevels do iScalar = 1,num_scalars horiz_flux_arr(iScalar,k,iEdge) = & - scalar_weight2(k,1) * scalar_new(iScalar,k,ica(1)) + & - scalar_weight2(k,2) * scalar_new(iScalar,k,ica(2)) + & - scalar_weight2(k,3) * scalar_new(iScalar,k,ica(3)) + & - scalar_weight2(k,4) * scalar_new(iScalar,k,ica(4)) + & - scalar_weight2(k,5) * scalar_new(iScalar,k,ica(5)) + & - scalar_weight2(k,6) * scalar_new(iScalar,k,ica(6)) + & - scalar_weight2(k,7) * scalar_new(iScalar,k,ica(7)) + & - scalar_weight2(k,8) * scalar_new(iScalar,k,ica(8)) + & - scalar_weight2(k,9) * scalar_new(iScalar,k,ica(9)) + & - scalar_weight2(k,10) * scalar_new(iScalar,k,ica(10)) + scalar_weight2(k,1) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & + scalar_weight2(k,2) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & + scalar_weight2(k,3) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & + scalar_weight2(k,4) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & + scalar_weight2(k,5) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & + scalar_weight2(k,6) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & + scalar_weight2(k,7) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & + scalar_weight2(k,8) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & + scalar_weight2(k,9) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & + scalar_weight2(k,10) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) end do end do From 5b83a01ac77cffbba078c5c83ae8d11797d25ed2 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jun 2026 17:55:59 -0600 Subject: [PATCH 4/6] Opt 3 - Case 10 Remove scalar_weight2 loop and inline it in next loop --- .../dynamics/mpas_atm_time_integration.F | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 9fea1c1545..169c1a1864 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4758,7 +4758,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & call mpas_timer_start('atm_advance_scalars_4760') !$acc parallel async - !$acc loop gang worker private(scalar_weight2) + !$acc loop gang worker do iEdge=edgeStart,edgeEnd if ((.not.config_apply_lbcs) & @@ -4768,27 +4768,20 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & case(10) - !$acc loop vector collapse(2) - do j=1,10 - do k=1,nVertLevels - scalar_weight2(k,j) = adv_coefs(j,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(j,iEdge) - end do - end do - !$acc loop vector collapse(2) do k = 1,nVertLevels do iScalar = 1,num_scalars horiz_flux_arr(iScalar,k,iEdge) = & - scalar_weight2(k,1) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & - scalar_weight2(k,2) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & - scalar_weight2(k,3) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & - scalar_weight2(k,4) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & - scalar_weight2(k,5) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & - scalar_weight2(k,6) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & - scalar_weight2(k,7) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & - scalar_weight2(k,8) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & - scalar_weight2(k,9) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & - scalar_weight2(k,10) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) + adv_coefs(1,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(1,iEdge) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & + adv_coefs(2,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(2,iEdge) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & + adv_coefs(3,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(3,iEdge) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & + adv_coefs(4,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(4,iEdge) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & + adv_coefs(5,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(5,iEdge) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & + adv_coefs(6,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(6,iEdge) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & + adv_coefs(7,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(7,iEdge) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & + adv_coefs(8,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(8,iEdge) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & + adv_coefs(9,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(9,iEdge) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & + adv_coefs(10,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(10,iEdge) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) end do end do From 102b6ad3850ef597c70db5271c4c2830aa09f216 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jun 2026 18:52:11 -0600 Subject: [PATCH 5/6] fix to prev commit --- .../dynamics/mpas_atm_time_integration.F | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 169c1a1864..f23d7946db 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4772,16 +4772,16 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & do k = 1,nVertLevels do iScalar = 1,num_scalars horiz_flux_arr(iScalar,k,iEdge) = & - adv_coefs(1,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(1,iEdge) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & - adv_coefs(2,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(2,iEdge) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & - adv_coefs(3,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(3,iEdge) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & - adv_coefs(4,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(4,iEdge) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & - adv_coefs(5,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(5,iEdge) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & - adv_coefs(6,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(6,iEdge) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & - adv_coefs(7,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(7,iEdge) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & - adv_coefs(8,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(8,iEdge) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & - adv_coefs(9,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(9,iEdge) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & - adv_coefs(10,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(10,iEdge) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) + (adv_coefs(1,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(1,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & + (adv_coefs(2,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(2,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & + (adv_coefs(3,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(3,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & + (adv_coefs(4,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(4,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & + (adv_coefs(5,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(5,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & + (adv_coefs(6,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(6,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & + (adv_coefs(7,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(7,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & + (adv_coefs(8,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(8,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & + (adv_coefs(9,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(9,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & + (adv_coefs(10,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(10,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) end do end do From 755e0fc04f68d3d2ff0db8ca00de694bc12ce33d Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jun 2026 19:58:41 -0600 Subject: [PATCH 6/6] Opt 4 - use local_sign --- .../dynamics/mpas_atm_time_integration.F | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index f23d7946db..8b26643538 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4725,7 +4725,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & real (kind=RKIND) :: weight_time_old, weight_time_new real (kind=RKIND), dimension(num_scalars,nVertLevels) :: scalar_tend_column ! local storage to accumulate tendency - real (kind=RKIND) :: u_direction, u_positive, u_negative + real (kind=RKIND) :: u_direction, u_positive, u_negative, local_sign flux4(q_im2, q_im1, q_i, q_ip1, ua) = & ua*( 7.*(q_i + q_im1) - (q_ip1 + q_im2) )/12.0 @@ -4755,7 +4755,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & if(rk_step == 3) weight_time_new = 1. end if weight_time_old = 1. - weight_time_new - + local_sign = 1.0_RKIND call mpas_timer_start('atm_advance_scalars_4760') !$acc parallel async !$acc loop gang worker @@ -4771,17 +4771,18 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & !$acc loop vector collapse(2) do k = 1,nVertLevels do iScalar = 1,num_scalars + local_sign = sign(1.0_RKIND,uhAvg(k,iEdge)) horiz_flux_arr(iScalar,k,iEdge) = & - (adv_coefs(1,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(1,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & - (adv_coefs(2,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(2,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & - (adv_coefs(3,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(3,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & - (adv_coefs(4,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(4,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & - (adv_coefs(5,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(5,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & - (adv_coefs(6,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(6,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & - (adv_coefs(7,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(7,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & - (adv_coefs(8,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(8,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & - (adv_coefs(9,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(9,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & - (adv_coefs(10,iEdge) + sign(1.0_RKIND,uhAvg(k,iEdge))*adv_coefs_3rd(10,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) + (adv_coefs(1,iEdge) + local_sign*adv_coefs_3rd(1,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(1,iEdge)) + & + (adv_coefs(2,iEdge) + local_sign*adv_coefs_3rd(2,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(2,iEdge)) + & + (adv_coefs(3,iEdge) + local_sign*adv_coefs_3rd(3,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(3,iEdge)) + & + (adv_coefs(4,iEdge) + local_sign*adv_coefs_3rd(4,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(4,iEdge)) + & + (adv_coefs(5,iEdge) + local_sign*adv_coefs_3rd(5,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(5,iEdge)) + & + (adv_coefs(6,iEdge) + local_sign*adv_coefs_3rd(6,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(6,iEdge)) + & + (adv_coefs(7,iEdge) + local_sign*adv_coefs_3rd(7,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(7,iEdge)) + & + (adv_coefs(8,iEdge) + local_sign*adv_coefs_3rd(8,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(8,iEdge)) + & + (adv_coefs(9,iEdge) + local_sign*adv_coefs_3rd(9,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(9,iEdge)) + & + (adv_coefs(10,iEdge) + local_sign*adv_coefs_3rd(10,iEdge)) * scalar_new(iScalar,k,advCellsForEdge(10,iEdge)) end do end do