Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More lazy fixes #2450

Merged
merged 4 commits into from
Jun 27, 2024
Merged

More lazy fixes #2450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b276f84
More lazy fixes
bouweandela Jun 10, 2024
1f9a25f
Fix lint
bouweandela Jun 10, 2024
45bc55a
Add note about bad data
bouweandela Jun 27, 2024
eebfc2f
Avoid line too long
bouweandela Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 15 additions & 58 deletions esmvalcore/cmor/_fixes/cmip5/ec_earth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,18 @@ class Sic(Fix):
"""Fixes for sic."""

def fix_data(self, cube):
"""
Fix data.
"""Fix data.

Fixes discrepancy between declared units and real units

Parameters
----------
cube: iris.cube.Cube
Cube to fix.

Returns
-------
iris.cube.Cube

"""
metadata = cube.metadata
cube *= 100
Expand All @@ -35,19 +34,18 @@ class Sftlf(Fix):
"""Fixes for sftlf."""

def fix_data(self, cube):
"""
Fix data.
"""Fix data.

Fixes discrepancy between declared units and real units

Parameters
----------
cube: iris.cube.Cube
Cube to fix.

Returns
-------
iris.cube.Cube

"""
metadata = cube.metadata
cube *= 100
Expand All @@ -59,19 +57,18 @@ class Tos(Fix):
"""Fixes for tos."""

def fix_data(self, cube):
"""
Fix tos data.
"""Fix tos data.

Fixes mask

Parameters
----------
cube: iris.cube.Cube
Cube to fix.

Returns
-------
iris.cube.Cube

"""
cube.data = da.ma.masked_equal(cube.core_data(), 273.15)
return cube
Expand All @@ -81,8 +78,7 @@ class Tas(Fix):
"""Fixes for tas."""

def fix_metadata(self, cubes):
"""
Fix potentially missing scalar dimension.
"""Fix potentially missing scalar dimension.

Parameters
----------
Expand All @@ -92,9 +88,7 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.CubeList

"""

for cube in cubes:
if not cube.coords(var_name='height'):
add_scalar_height_coord(cube)
Expand All @@ -109,8 +103,7 @@ class Areacello(Fix):
"""Fixes for areacello."""

def fix_metadata(self, cubes):
"""
Fix potentially missing scalar dimension.
"""Fix potentially missing scalar dimension.

Parameters
----------
Expand All @@ -120,7 +113,6 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.CubeList

"""
areacello = cubes.extract('Areas of grid cell')[0]
lat = cubes.extract('latitude')[0]
Expand All @@ -129,7 +121,9 @@ def fix_metadata(self, cubes):
areacello.add_aux_coord(cube_to_aux_coord(lat), (0, 1))
areacello.add_aux_coord(cube_to_aux_coord(lon), (0, 1))

return iris.cube.CubeList([areacello, ])
return iris.cube.CubeList([
areacello,
])


class Pr(Fix):
Expand All @@ -150,56 +144,19 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.CubeList

"""
new_list = iris.cube.CubeList()
for cube in cubes:
try:
old_time = cube.coord('time')
time_coord = cube.coord('time')
except iris.exceptions.CoordinateNotFoundError:
new_list.append(cube)
else:
if old_time.is_monotonic():
if time_coord.is_monotonic():
new_list.append(cube)
else:
time_units = old_time.units
time_data = old_time.points

# erase erroneously copy-pasted points
time_diff = np.diff(time_data)
idx_neg = np.where(time_diff <= 0.)[0]
while len(idx_neg) > 0:
time_data = np.delete(time_data, idx_neg[0] + 1)
time_diff = np.diff(time_data)
idx_neg = np.where(time_diff <= 0.)[0]

# create the new time coord
new_time = iris.coords.DimCoord(time_data,
standard_name='time',
var_name='time',
units=time_units)

# create a new cube with the right shape
dims = (time_data.shape[0],
cube.coord('latitude').shape[0],
cube.coord('longitude').shape[0])
data = cube.data
new_data = np.ma.append(data[:dims[0] - 1, :, :],
data[-1, :, :])
new_data = new_data.reshape(dims)

tmp_cube = iris.cube.Cube(
new_data,
standard_name=cube.standard_name,
long_name=cube.long_name,
var_name=cube.var_name,
units=cube.units,
attributes=cube.attributes,
cell_methods=cube.cell_methods,
dim_coords_and_dims=[(new_time, 0),
(cube.coord('latitude'), 1),
(cube.coord('longitude'), 2)])

new_list.append(tmp_cube)
select = np.unique(time_coord.points, return_index=True)[1]
new_list.append(cube[select])

return new_list
68 changes: 49 additions & 19 deletions esmvalcore/cmor/_fixes/cmip6/access_esm1_5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,51 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.CubeList

"""
for cube in cubes:
try:
bcoeff = cube.coord(var_name='b')
# values taken from HadGEM2-ES model (CMIP5), which uses the
# same atmospheric component as ACCESS-ESM1-5 (HadGAM2, N96L38)
bcoeff.points = [
0.99771648645401, 0.990881502628326, 0.979542553424835,
0.9637770652771, 0.943695485591888, 0.919438362121582,
0.891178011894226, 0.859118342399597, 0.823493480682373,
0.784570515155792, 0.742646217346191, 0.698050200939178,
0.651142716407776, 0.602314412593842, 0.55198872089386,
0.500619947910309, 0.44869339466095, 0.39672577381134,
0.34526526927948, 0.294891387224197, 0.24621507525444,
0.199878215789795, 0.156554222106934, 0.116947874426842,
0.0817952379584312, 0.0518637150526047, 0.0279368180781603,
0.0107164792716503, 0.00130179093685001,
0, 0, 0, 0, 0, 0, 0, 0, 0,
0.99771648645401,
0.990881502628326,
0.979542553424835,
0.9637770652771,
0.943695485591888,
0.919438362121582,
0.891178011894226,
0.859118342399597,
0.823493480682373,
0.784570515155792,
0.742646217346191,
0.698050200939178,
0.651142716407776,
0.602314412593842,
0.55198872089386,
0.500619947910309,
0.44869339466095,
0.39672577381134,
0.34526526927948,
0.294891387224197,
0.24621507525444,
0.199878215789795,
0.156554222106934,
0.116947874426842,
0.0817952379584312,
0.0518637150526047,
0.0279368180781603,
0.0107164792716503,
0.00130179093685001,
0,
0,
0,
0,
0,
0,
0,
0,
0,
]
bcoeff.bounds = [
[1, 0.994296252727509],
Expand Down Expand Up @@ -70,8 +96,15 @@ def fix_metadata(self, cubes):
[0.0389823913574219, 0.0183146875351667],
[0.0183146875351667, 0.00487210927531123],
[0.00487210927531123, 0],
[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0],
[0, 0], [0, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 0],
]
except iris.exceptions.CoordinateNotFoundError:
pass
Expand All @@ -80,7 +113,6 @@ def fix_metadata(self, cubes):

Cli = Cl


Clw = Cl


Expand All @@ -98,7 +130,6 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.Cube

"""
cube = self.get_cube_from_list(cubes)
cube.coord('air_pressure').points = \
Expand All @@ -122,11 +153,10 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.Cube

"""
cube = self.get_cube_from_list(cubes)
cube.coord('air_pressure').points = \
np.round(cube.coord('air_pressure').points, 0)
np.round(cube.coord('air_pressure').core_points(), 0)
cube.coord('air_pressure').bounds = \
np.round(cube.coord('air_pressure').bounds, 0)
np.round(cube.coord('air_pressure').core_bounds(), 0)
return cubes
6 changes: 2 additions & 4 deletions esmvalcore/cmor/_fixes/cmip6/cnrm_cm6_1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
from ..fix import Fix
from ..shared import (
add_aux_coords_from_cubes,
fix_ocean_depth_coord,
get_bounds_cube,
fix_ocean_depth_coord
)


Expand All @@ -24,7 +24,6 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.Cube

"""
cube = self.get_cube_from_list(cubes)

Expand All @@ -40,7 +39,7 @@ def fix_metadata(self, cubes):
# Fix vertical coordinate bounds
for coord_name in ('ap', 'b'):
bounds_cube = get_bounds_cube(cubes, coord_name)
bounds = bounds_cube.data.reshape(-1, 2)
bounds = bounds_cube.core_data().reshape(-1, 2)
new_bounds_cube = iris.cube.Cube(bounds,
**bounds_cube.metadata._asdict())
cubes.remove(bounds_cube)
Expand Down Expand Up @@ -91,7 +90,6 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.CubeList

"""
for cube in cubes:
if cube.coords(axis='Z'):
Expand Down
21 changes: 14 additions & 7 deletions esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Fixes for FGOALS-g3 model."""
import dask.array as da
import iris

from ..common import OceanFixGrid
Expand All @@ -22,7 +23,13 @@ def fix_metadata(self, cubes):
"""Fix metadata.

FGOALS-g3 data contain latitude and longitude data set to >1e30 in some
places.
places. Note that the corresponding data is all masked.

Example files:
v20191030/siconc_SImon_FGOALS-g3_piControl_r1i1p1f1_gn_070001-079912.nc
v20191030/siconc_SImon_FGOALS-g3_piControl_r1i1p1f1_gn_080001-089912.nc
v20200706/siconc_SImon_FGOALS-g3_ssp534-over_r1i1p1f1_gn_201501-210012
.nc

Parameters
----------
Expand All @@ -32,13 +39,13 @@ def fix_metadata(self, cubes):
Returns
-------
iris.cube.CubeList

"""
cube = self.get_cube_from_list(cubes)
cube.coord('latitude').points[
cube.coord('latitude').points > 1000.0] = 0.0
cube.coord('longitude').points[
cube.coord('longitude').points > 1000.0] = 0.0
for coord_name in ['latitude', 'longitude']:
coord = cube.coord(coord_name)
bad_indices = coord.core_points() > 1000.0
coord.points = da.ma.where(bad_indices, 0.0, coord.core_points())
schlunma marked this conversation as resolved.
Show resolved Hide resolved

return super().fix_metadata(cubes)


Expand All @@ -51,7 +58,7 @@ class Mrsos(Fix):
def fix_metadata(self, cubes):
"""Fix metadata.

FGOALS-g3 mrsos data contains error in co-ordinate bounds.
FGOALS-g3 mrsos data contains error in coordinate bounds.
bouweandela marked this conversation as resolved.
Show resolved Hide resolved

Parameters
----------
Expand Down
Loading