Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert storage temporaries in fv3core to Quantity #364

Merged
merged 22 commits into from
Nov 2, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
2be4961
convert temporaries on dycore, dyn_core, and d_sw to quantity
mcgibbon Oct 21, 2022
b29adcf
convert all fv3core temporaries to quantity
mcgibbon Oct 21, 2022
fa4a040
fix some mpi parallel tests (some still failing)
mcgibbon Oct 21, 2022
a988bf6
fix remapping, main tests
mcgibbon Oct 24, 2022
f9e8c3d
fix parallel tests
mcgibbon Oct 24, 2022
8ffc40b
delete dead code
mcgibbon Oct 24, 2022
79b6875
allow diagnostics output directory to already exist
mcgibbon Oct 25, 2022
c4cba2d
clean up usage of dp_ref
mcgibbon Oct 26, 2022
1c42eb3
fix calculation of pfull in fv_dynamics
mcgibbon Oct 26, 2022
20c0445
add buffer point to dp_ref in updatedzd
mcgibbon Oct 26, 2022
dcbdf86
fix type of phis in fv_dynamics test
mcgibbon Oct 26, 2022
3d129f5
Merge branch 'main' into feature/only_quantity
mcgibbon Oct 27, 2022
ca70d8e
attempt to fix some gpu data errors
mcgibbon Oct 27, 2022
883a212
use q.np.asarray instead of q.asarray which does not exist
mcgibbon Oct 27, 2022
8b7728c
Merge branch 'main' into feature/only_quantity
mcgibbon Oct 28, 2022
feca290
cast np/cp arrays when assigning from savepoint data
mcgibbon Oct 28, 2022
8963070
Empty commit to fix daint grabbing wrong code
mcgibbon Oct 28, 2022
28a7d6e
cast to correct ndarray in remaining test
mcgibbon Oct 31, 2022
cb1fe46
Merge branch 'main' into feature/only_quantity
mcgibbon Oct 31, 2022
2c681bb
Merge branch 'main' into feature/only_quantity
elynnwu Nov 2, 2022
8109bd5
Merge branch 'main' into feature/only_quantity
mcgibbon Nov 2, 2022
1c49a9c
Merge branch 'feature/only_quantity' of github.com:ai2cm/pace into fe…
mcgibbon Nov 2, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion driver/examples/configs/tropical_read_restart_fortran.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ dycore_config:
tau_v2l: 150.
fv_sg_adj: 600
n_sponge: 48
p_ref: 101500.

physics_config:
hydrostatic: false
Expand Down
2 changes: 1 addition & 1 deletion driver/pace/driver/diagnostics.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def diagnostics_factory(self, communicator: pace.util.Communicator) -> Diagnosti
else:
fs = pace.util.get_fs(self.path)
if not fs.exists(self.path):
fs.makedirs(self.path)
fs.makedirs(self.path, exist_ok=True)
if self.output_format == "zarr":
store = zarr_storage.DirectoryStore(path=self.path)
monitor: pace.util.Monitor = pace.util.ZarrMonitor(
Expand Down
2 changes: 2 additions & 0 deletions driver/pace/driver/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,7 @@ def exit_instead_of_build(self):
comm=communicator,
grid_data=self.state.grid_data,
stencil_factory=self.stencil_factory,
quantity_factory=self.quantity_factory,
damping_coefficients=self.state.damping_coefficients,
config=self.config.dycore_config,
timestep=self.config.timestep,
Expand All @@ -491,6 +492,7 @@ def exit_instead_of_build(self):
if not config.disable_step_physics:
self.dycore_to_physics = update_atmos_state.DycoreToPhysics(
stencil_factory=self.stencil_factory,
quantity_factory=self.quantity_factory,
dycore_config=self.config.dycore_config,
do_dry_convective_adjust=config.do_dry_convective_adjustment,
dycore_only=self.config.dycore_only,
Expand Down
2 changes: 1 addition & 1 deletion driver/pace/driver/initialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ def _update_fortran_restart_pe_peln(state: DriverState) -> None:
and updates the driver state with values.
"""

ptop = state.grid_data._vertical_data.ak.view[0]
ptop = state.grid_data.ak.view[0]
pe = state.dycore_state.pe
peln = state.dycore_state.peln
delp = state.dycore_state.delp
Expand Down
2 changes: 1 addition & 1 deletion dsl/pace/dsl/stencil.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def report_diff(arg: np.ndarray, numpy_arg: np.ndarray, label) -> str:
failures_8 = n_points - np.sum(
np.logical_or(
nans_match,
metric_err < 1e-10,
metric_err < 1e-8,
)
)
greatest_error = np.max(metric_err[~np.isnan(metric_err)])
Expand Down
1 change: 0 additions & 1 deletion fv3core/examples/standalone/runfile/acoustics.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,6 @@ def driver(
nested,
stretched_grid,
dycore_config.acoustic_dynamics,
input_data["pfull"],
input_data["phis"],
input_data["wsd"],
state,
Expand Down
4 changes: 0 additions & 4 deletions fv3core/pace/fv3core/_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,9 +267,6 @@ class DynamicalCoreConfig:
breed_vortex_inline: bool = NamelistDefaults.breed_vortex_inline
use_old_omega: bool = NamelistDefaults.use_old_omega
rf_fast: bool = NamelistDefaults.rf_fast
p_ref: float = (
NamelistDefaults.p_ref
) # Surface pressure used to construct a horizontally-uniform reference
adiabatic: bool = NamelistDefaults.adiabatic
nf_omega: int = NamelistDefaults.nf_omega
fv_sg_adj: int = NamelistDefaults.fv_sg_adj
Expand Down Expand Up @@ -375,7 +372,6 @@ def from_namelist(cls, namelist: Namelist) -> "DynamicalCoreConfig":
breed_vortex_inline=namelist.breed_vortex_inline,
use_old_omega=namelist.use_old_omega,
rf_fast=namelist.rf_fast,
p_ref=namelist.p_ref,
adiabatic=namelist.adiabatic,
nf_omega=namelist.nf_omega,
fv_sg_adj=namelist.fv_sg_adj,
Expand Down
22 changes: 13 additions & 9 deletions fv3core/pace/fv3core/stencils/a2b_ord4.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
sqrt,
)

import pace.dsl.gt4py_utils as utils
import pace.util
from pace.dsl.dace.orchestration import orchestrate
from pace.dsl.stencil import GridIndexing, StencilFactory
from pace.dsl.typing import FloatField, FloatFieldI, FloatFieldIJ
Expand Down Expand Up @@ -532,6 +532,7 @@ class AGrid2BGridFourthOrder:
def __init__(
self,
stencil_factory: StencilFactory,
quantity_factory: pace.util.QuantityFactory,
grid_data: GridData,
grid_type,
z_dim=Z_DIM,
Expand Down Expand Up @@ -564,15 +565,18 @@ def __init__(

self.replace = replace

def make_storage():
return utils.make_storage_from_shape(
self._idx.max_shape,
backend=self._stencil_config.compilation_config.backend,
)
self._tmp_qx = quantity_factory.zeros(
dims=[X_INTERFACE_DIM, Y_DIM, z_dim], units="unknown"
)
self._tmp_qy = quantity_factory.zeros(
dims=[X_DIM, Y_INTERFACE_DIM, z_dim], units="unknown"
)
# TODO: the dimensions of tmp_qout_edges may not be correct, verify
# with Lucas and either update the code or remove this comment
self._tmp_qout_edges = quantity_factory.zeros(
dims=[X_DIM, Y_DIM, z_dim], units="unknown"
)

self._tmp_qx = make_storage()
self._tmp_qy = make_storage()
self._tmp_qout_edges = make_storage()
_, (z_domain,) = self._idx.get_origin_domain([z_dim])
corner_domain = (1, 1, z_domain)

Expand Down
14 changes: 1 addition & 13 deletions fv3core/pace/fv3core/stencils/basic_operations.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import gt4py.gtscript as gtscript
from gt4py.gtscript import FORWARD, PARALLEL, computation, cos, interval, sin
from gt4py.gtscript import PARALLEL, computation, interval

from pace.dsl.typing import FloatField, FloatFieldIJ
from pace.util.constants import OMEGA


def copy_defn(q_in: FloatField, q_out: FloatField):
Expand Down Expand Up @@ -31,17 +30,6 @@ def adjust_divide_stencil(adjustment: FloatField, q_out: FloatField):
q_out = q_out / adjustment


def compute_coriolis_parameter_defn(
f: FloatFieldIJ, lon: FloatFieldIJ, lat: FloatFieldIJ, alpha: float
):
with computation(FORWARD), interval(0, 1):
f = (
2.0
* OMEGA
* (-1.0 * cos(lon) * cos(lat) * sin(alpha) + sin(lat) * cos(alpha))
)


@gtscript.function
def sign(a, b):
asignb = abs(a)
Expand Down
71 changes: 21 additions & 50 deletions fv3core/pace/fv3core/stencils/c_sw.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@
region,
)

import pace.dsl.gt4py_utils as utils
import pace.util
from pace.dsl.dace.orchestration import orchestrate
from pace.dsl.stencil import StencilFactory
from pace.dsl.typing import FloatField, FloatFieldIJ
from pace.fv3core.stencils.basic_operations import compute_coriolis_parameter_defn
from pace.fv3core.stencils.d2a2c_vect import DGrid2AGrid2CGridVectors
from pace.stencils import corners
from pace.util import X_DIM, X_INTERFACE_DIM, Y_DIM, Y_INTERFACE_DIM, Z_DIM
Expand Down Expand Up @@ -481,22 +480,6 @@ def update_y_velocity(
velocity_c = velocity_c - tmp_flux * flux + rdyc * (ke[0, -1, 0] - ke)


def compute_fC(
stencil_factory: StencilFactory, lon: FloatFieldIJ, lat: FloatFieldIJ, backend: str
):
"""
Compute the coriolis parameter on the C-grid
"""
fC = utils.make_storage_from_shape(lon.shape, backend=backend)
fC_stencil = stencil_factory.from_dims_halo(
compute_coriolis_parameter_defn,
compute_dims=[X_INTERFACE_DIM, Y_INTERFACE_DIM, Z_DIM],
compute_halos=(3, 3),
)
fC_stencil(fC, lon, lat, 0.0)
return fC


class CGridShallowWaterDynamics:
"""
Fortran name is c_sw
Expand All @@ -505,35 +488,23 @@ class CGridShallowWaterDynamics:
def __init__(
self,
stencil_factory: StencilFactory,
quantity_factory: pace.util.QuantityFactory,
grid_data: GridData,
nested: bool,
grid_type: int,
nord: int,
):
orchestrate(obj=self, config=stencil_factory.config.dace_config)
grid_indexing = stencil_factory.grid_indexing
self.grid_data = grid_data
self._dord4 = True
self._fC = compute_fC(
stencil_factory,
self.grid_data.lon,
self.grid_data.lat,
backend=stencil_factory.backend,
)
origin_halo1 = (grid_indexing.isc - 1, grid_indexing.jsc - 1, 0)
self.delpc = utils.make_storage_from_shape(
grid_indexing.max_shape,
origin=origin_halo1,
backend=stencil_factory.backend,
)
self._fC = self.grid_data.fC
# TODO: double-check the dimensions on these, they may be incorrect
# as they are only documentation and not used by the code
self.delpc = quantity_factory.zeros([X_DIM, Y_DIM, Z_DIM], units="unknown")
"""
pressure thickness on c-grid forward step
"""
self.ptc = utils.make_storage_from_shape(
grid_indexing.max_shape,
origin=origin_halo1,
backend=stencil_factory.backend,
)
self.ptc = quantity_factory.zeros([X_DIM, Y_DIM, Z_DIM], units="unknown")
"""
potential temperature on c-grid forward step
"""
Expand All @@ -545,23 +516,23 @@ def __init__(

self._D2A2CGrid_Vectors = DGrid2AGrid2CGridVectors(
stencil_factory,
grid_data,
nested,
grid_type,
self._dord4,
quantity_factory=quantity_factory,
grid_data=grid_data,
nested=nested,
grid_type=grid_type,
dord4=self._dord4,
)

def make_storage():
return utils.make_storage_from_shape(
grid_indexing.max_shape,
backend=stencil_factory.backend,
)
def make_quantity() -> pace.util.Quantity:
return quantity_factory.zeros([X_DIM, Y_DIM, Z_DIM], units="unknown")

self._tmp_ke = make_storage()
self._tmp_vort = make_storage()
self._tmp_fx = make_storage()
self._tmp_fx1 = make_storage()
self._tmp_fx2 = make_storage()
# TODO: double-check the dimensions on these, they may be incorrect
# as they are only documentation and not used by the code
self._tmp_ke = make_quantity()
self._tmp_vort = make_quantity()
self._tmp_fx = make_quantity()
self._tmp_fx1 = make_quantity()
self._tmp_fx2 = make_quantity()

if nord > 0:
self._divergence_corner = stencil_factory.from_dims_halo(
Expand Down
15 changes: 4 additions & 11 deletions fv3core/pace/fv3core/stencils/d2a2c_vect.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import gt4py.gtscript as gtscript
from gt4py.gtscript import PARALLEL, computation, horizontal, interval, region

import pace.dsl.gt4py_utils as utils
import pace.util
from pace.dsl.dace.orchestration import orchestrate
from pace.dsl.stencil import StencilFactory
from pace.dsl.typing import FloatField, FloatFieldIJ
Expand Down Expand Up @@ -385,6 +385,7 @@ class DGrid2AGrid2CGridVectors:
def __init__(
self,
stencil_factory: StencilFactory,
quantity_factory: pace.util.QuantityFactory,
grid_data: GridData,
nested: bool,
grid_type: int,
Expand Down Expand Up @@ -418,16 +419,8 @@ def __init__(
npt = 4 if not nested else 0
if npt > grid_indexing.domain[0] - 1 or npt > grid_indexing.domain[1] - 1:
npt = 0
self._utmp = utils.make_storage_from_shape(
grid_indexing.max_shape,
grid_indexing.origin_full(),
backend=stencil_factory.backend,
)
self._vtmp = utils.make_storage_from_shape(
grid_indexing.max_shape,
grid_indexing.origin_full(),
backend=stencil_factory.backend,
)
self._utmp = quantity_factory.zeros([X_DIM, Y_DIM, Z_DIM], units="m/s")
self._vtmp = quantity_factory.zeros([X_DIM, Y_DIM, Z_DIM], units="m/s")

js1 = npt + OFFSET if grid_indexing.south_edge else grid_indexing.jsc - 1
je1 = ny - npt if grid_indexing.north_edge else grid_indexing.jec + 1
Expand Down
Loading