OpenMP: PGF: Copy PGF outside of function

Moving the map(from:) call outside of the function.
Also create PressureForce_FV CS during initialization.

This is just a start, going to add more to this commit.

src/core/MOM_PressureForce_FV.F90

@@ -1607,13 +1607,9 @@ subroutine PressureForce_FV_Bouss(h, tv, PFu, PFv, G, GV, US, CS, ALE_CSp, p_atm
   !$acc   copyin(pa, h, intx_pa, inty_pa, intx_dpa, inty_dpa, intz_dpa) &
   !$acc   create(rho_in_situ, dM)
 
-  ! This should certainly be outside of the function
-  !$omp target enter data map(to: CS)
-
   !$omp target enter data &
-  !$omp   map(to: CS, pa, h, e) &
-  !$omp   map(to: intx_pa, inty_pa, intx_dpa, inty_dpa, intz_dpa) &
-  !$omp   map(alloc: PFu, PFv)
+  !$omp   map(to: pa, h, e) &
+  !$omp   map(to: intx_pa, inty_pa, intx_dpa, inty_dpa, intz_dpa)
 
   !$omp target enter data if(use_EOS) &
   !$omp   map(to: tv_tmp, tv_tmp%T, tv_tmp%S, tv, tv%eqn_of_state, EOSdom2d)
@@ -1725,8 +1721,7 @@ subroutine PressureForce_FV_Bouss(h, tv, PFu, PFv, G, GV, US, CS, ALE_CSp, p_atm
 
   !$omp target exit data &
   !$omp   map(delete: pa, h) &
-  !$omp   map(delete: intx_pa, inty_pa, intx_dpa, inty_dpa, intz_dpa) &
-  !$omp   map(from: PFu, PFv)
+  !$omp   map(delete: intx_pa, inty_pa, intx_dpa, inty_dpa, intz_dpa)
 
   ! NOTE: As above, these are here until data is set up outside of the function.
 
@@ -1755,7 +1750,7 @@ subroutine PressureForce_FV_Bouss(h, tv, PFu, PFv, G, GV, US, CS, ALE_CSp, p_atm
     !   Most likely a bug but innocuous since e_sal is initialized to zero.
     !   This has been fixed by He in a PR, but for now we use it as-is.
     !$omp target enter data &
-    !$omp   if((CS%Calculate_SAL .and. CS%tides_answer_date > 20230630) .or. .not. CS%tides) &
+    !$omp   if((CS%calculate_SAL .and. CS%tides_answer_date > 20230630) .or. .not. CS%tides) &
     !$omp   map(to: e_sal)
 
     !$omp target enter data map(alloc: eta)
@@ -1806,6 +1801,7 @@ subroutine PressureForce_FV_Bouss(h, tv, PFu, PFv, G, GV, US, CS, ALE_CSp, p_atm
     !$acc exit data delete(e)
     !$acc exit data copyout(eta)
 
+    ! TODO: These may be needed for the diagnostics below!
     !$omp target exit data if(CS%tides .and. CS%tides_answer_date > 20230630) &
     !$omp   map(delete: e_tide_eq, e_tide_sal)
     !$omp target exit data if(CS%tides .and. CS%tides_answer_date <= 20230630) &
@@ -1817,7 +1813,7 @@ subroutine PressureForce_FV_Bouss(h, tv, PFu, PFv, G, GV, US, CS, ALE_CSp, p_atm
     !$omp target exit data map(from: eta)
   endif
 
-  !$omp target exit data map(delete: CS, e)
+  !$omp target exit data map(delete: e)
 
   if (CS%use_stanley_pgf) then
     ! Calculated diagnostics related to the Stanley parameterization

src/core/MOM_dynamics_split_RK2.F90

@@ -492,6 +492,7 @@ subroutine step_MOM_dyn_split_RK2(u_inst, v_inst, h, tv, visc, Time_local, dt, f
   call cpu_clock_begin(id_clock_pres)
   call PressureForce(h, tv, CS%PFu, CS%PFv, G, GV, US, CS%PressureForce_CSp, &
                      CS%ALE_CSp, p_surf, CS%pbce, CS%eta_PF)
+  !$omp target update from(CS%PFu, CS%PFv)
   if (dyn_p_surf) then
     pres_to_eta = 1.0 / (GV%g_Earth * GV%H_to_RZ)
     !$OMP parallel do default(shared)
@@ -811,6 +812,7 @@ subroutine step_MOM_dyn_split_RK2(u_inst, v_inst, h, tv, visc, Time_local, dt, f
     call cpu_clock_begin(id_clock_pres)
     call PressureForce(hp, tv, CS%PFu, CS%PFv, G, GV, US, CS%PressureForce_CSp, &
                        CS%ALE_CSp, p_surf, CS%pbce, CS%eta_PF)
+    !$omp target exit data map(from: CS%PFu, CS%PFv)
     ! Stokes shear force contribution to pressure gradient
     Use_Stokes_PGF = present(Waves)
     if (Use_Stokes_PGF) then
@@ -1222,7 +1224,8 @@ subroutine register_restarts_dyn_split_RK2(HI, GV, US, param_file, CS, restart_C
   ALLOC_(CS%CAv_pred(isd:ied,JsdB:JedB,nz)) ; CS%CAv_pred(:,:,:)   = 0.0
   ALLOC_(CS%PFu(IsdB:IedB,jsd:jed,nz))   ; CS%PFu(:,:,:)   = 0.0
   ALLOC_(CS%PFv(isd:ied,JsdB:JedB,nz))   ; CS%PFv(:,:,:)   = 0.0
-
+  ! TODO: Replace with map(alloc:) (and remove PF[uv] = 0.)
+  !$omp target enter data map(to: CS%PFu, CS%PFv)
   ALLOC_(CS%eta(isd:ied,jsd:jed))       ; CS%eta(:,:)    = 0.0
   ALLOC_(CS%u_av(IsdB:IedB,jsd:jed,nz)) ; CS%u_av(:,:,:) = 0.0
   ALLOC_(CS%v_av(isd:ied,JsdB:JedB,nz)) ; CS%v_av(:,:,:) = 0.0
@@ -1515,6 +1518,7 @@ subroutine initialize_dyn_split_RK2(u, v, h, tv, uh, vh, eta, Time, G, GV, US, p
   endif
   call PressureForce_init(Time, G, GV, US, param_file, diag, CS%PressureForce_CSp, &
                           CS%SAL_CSp, CS%tides_CSp)
+  !$omp target enter data map(to: CS%PressureForce_CSp)
   call hor_visc_init(Time, G, GV, US, param_file, diag, CS%hor_visc, ADp=CS%ADp)
   call vertvisc_init(MIS, Time, G, GV, US, param_file, diag, CS%ADp, dirs, ntrunc, CS%vertvisc_CSp)
   CS%set_visc_CSp => set_visc
@@ -1851,6 +1855,7 @@ subroutine end_dyn_split_RK2(CS)
   DEALLOC_(CS%CAu)   ; DEALLOC_(CS%CAv)
   DEALLOC_(CS%CAu_pred) ; DEALLOC_(CS%CAv_pred)
   DEALLOC_(CS%PFu)   ; DEALLOC_(CS%PFv)
+  !$omp target exit data map(delete: CS%PFu, CS%PFv)
 
   if (associated(CS%taux_bot)) deallocate(CS%taux_bot)
   if (associated(CS%tauy_bot)) deallocate(CS%tauy_bot)