Add acc data management to atm_recover_large_step_variables_work

These changes ensure that the other, non-invariant, fields are available
on the device during this routine. Some fields that are overwritten are
only created at the beginning, while others are copied in. Any fields
that were unmodified right-hand side fields are deleted at the end and
modified left-hand side fields are copied out.

Timing for these transfers are reported in the output log file in the
new timer: `atm_recover_large_step_variables [ACC_data_xfer]`.

'default(present)' clauses are added to the parallel directives so that
any missing data on the device causes a runtime error.

src/core_atmosphere/dynamics/mpas_atm_time_integration.F

@@ -2816,13 +2816,21 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE
       integer :: i, iCell, iEdge, k, cell1, cell2
       real (kind=RKIND) :: invNs, rcv, p0, flux
 
+      MPAS_ACC_TIMER_START('atm_recover_large_step_variables [ACC_data_xfer]')
+      !$acc enter data copyin(rho_p_save,rho_pp,rho_base,rw_save, &
+      !$acc                  rw_p,rtheta_base,rtheta_p_save,rtheta_pp, &
+      !$acc                  rt_diabatic_tend,exner_base,ru_save,ru_p,ruAvg)
+      !$acc enter data create(rho_zz,rho_p,rw,w,rtheta_p, &
+      !$acc                  theta_m,exner,pressure_p,ru,u)
+      !$acc enter data copyin(wwAvg,ruAvg)
+      MPAS_ACC_TIMER_STOP('atm_recover_large_step_variables [ACC_data_xfer]')
 
       rcv = rgas/(cp-rgas)
       p0 = 1.0e+05  ! this should come from somewhere else...
 
       ! Avoid FP errors caused by a potential division by zero below by 
       ! initializing the "garbage cell" of rho_zz to a non-zero value
-      !$acc parallel
+      !$acc parallel default(present)
       !$acc loop vector
       do k=1,nVertLevels
          rho_zz(k,nCells+1) = 1.0
@@ -2834,7 +2842,7 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE
 
       invNs = 1 / real(ns,RKIND)
 
-      !$acc parallel
+      !$acc parallel default(present)
       !$acc loop gang worker
       do iCell=cellStart,cellEnd
 
@@ -2864,7 +2872,7 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE
       !$acc end parallel
 
       if (rk_step == 3) then
-         !$acc parallel
+         !$acc parallel default(present)
          !$acc loop collapse(2)
          do iCell=cellStart,cellEnd
 !DIR$ IVDEP
@@ -2880,7 +2888,7 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE
          end do
          !$acc end parallel
       else
-         !$acc parallel
+         !$acc parallel default(present)
          !$acc loop collapse(2)
          do iCell=cellStart,cellEnd
 !DIR$ IVDEP
@@ -2899,7 +2907,7 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE
 
 !$OMP BARRIER
 
-      !$acc parallel
+      !$acc parallel default(present)
       !$acc loop gang worker
       do iEdge=edgeStart,edgeEnd
 
@@ -2960,6 +2968,15 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE
       end do
       !$acc end parallel
 
+      MPAS_ACC_TIMER_START('atm_recover_large_step_variables [ACC_data_xfer]')
+      !$acc exit data delete(rho_p_save,rho_pp,rho_base,rw_save, &
+      !$acc                  rw_p,rtheta_base,rtheta_p_save,rtheta_pp, &
+      !$acc                  rt_diabatic_tend,ru_save,ru_p,ruAvg)
+      !$acc exit data copyout(rho_zz,rho_p,rw,w,rtheta_p, &
+      !$acc                   theta_m,exner,pressure_p,ru,u)
+      !$acc exit data copyout(wwAvg,ruAvg)
+      MPAS_ACC_TIMER_STOP('atm_recover_large_step_variables [ACC_data_xfer]')
+
    end subroutine atm_recover_large_step_variables_work