From edd35fa2fb6016a4a5fc63e19af9ad66b4ef437a Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 24 Jan 2020 16:21:42 +0000 Subject: [PATCH 01/46] Attempt to copy in changes developed locally to the nimrod file format handler. --- lib/iris/fileformats/nimrod.py | 123 +++- lib/iris/fileformats/nimrod_load_rules.py | 639 +++++++++++++----- lib/iris/tests/integration/test_regridding.py | 4 +- lib/iris/tests/results/nimrod/load_2flds.cml | 26 +- .../results/nimrod/period_of_interest.cml | 3 +- lib/iris/tests/test_nimrod.py | 2 - .../nimrod_load_rules/test_vertical_coord.py | 7 - 7 files changed, 574 insertions(+), 230 deletions(-) diff --git a/lib/iris/fileformats/nimrod.py b/lib/iris/fileformats/nimrod.py index a2dede8ba7..2731c4fb27 100644 --- a/lib/iris/fileformats/nimrod.py +++ b/lib/iris/fileformats/nimrod.py @@ -48,7 +48,7 @@ "proj_biaxial_ellipsoid", "ensemble_member", "spare1", - "spare2", + "averagingtype", ) @@ -70,6 +70,8 @@ "true_origin_easting", "true_origin_northing", "tm_meridian_scaling", + "threshold_value_alt", + "threshold_value", ) @@ -87,46 +89,95 @@ "sat_space_count", "ducting_index", "elevation_angle", + 'neighbourhood_radius', + 'threshold_vicinity_radius', + 'recursive_filter_alpha', + 'threshold_fuzziness', + 'threshold_duration_fuzziness', + 'data_header_float32_05', + 'data_header_float32_06', + 'data_header_float32_07', + 'data_header_float32_08', + 'data_header_float32_09', + 'data_header_float32_10', + 'data_header_float32_11', + 'data_header_float32_12', + 'data_header_float32_13', + 'data_header_float32_14', + 'data_header_float32_15', + 'data_header_float32_16', + 'data_header_float32_17', + 'data_header_float32_18', + 'data_header_float32_19', + 'data_header_float32_20', + 'data_header_float32_21', + 'data_header_float32_22', + 'data_header_float32_23', + 'data_header_float32_24', + 'data_header_float32_25', + 'data_header_float32_26', + 'data_header_float32_27', + 'data_header_float32_28', + 'data_header_float32_29', + 'data_header_float32_30', + 'data_header_float32_31', + 'data_header_float32_32', ) # data specific header (int16) elements data_header_int16s = ( - "radar_num", - "radars_bitmask", - "more_radars_bitmask", - "clutter_map_num", - "calibration_type", - "bright_band_height", - "bright_band_intensity", - "bright_band_test1", - "bright_band_test2", - "infill_flag", - "stop_elevation", - "int16_vertical_coord", - "int16_reference_vertical_coord", - "int16_y_origin", - "int16_row_step", - "int16_x_origin", - "int16_column_step", - "int16_float32_mdi", - "int16_data_scaling", - "int16_data_offset", - "int16_x_offset", - "int16_y_offset", - "int16_true_origin_latitude", - "int16_true_origin_longitude", - "int16_tl_y", - "int16_tl_x", - "int16_tr_y", - "int16_ty_x", - "int16_br_y", - "int16_br_x", - "int16_bl_y", - "int16_bl_x", - "sensor_id", - "meteosat_id", - "alphas_available", + "threshold_type", + "probability_method", + "recursive_filter_iterations", + "member_count", + "probability_period_of_event", + "data_header_int16_05", + "soil_type", + "radiation_code", + "data_header_int16_08", + "data_header_int16_09", + "data_header_int16_10", + "data_header_int16_11", + "data_header_int16_12", + "data_header_int16_13", + "data_header_int16_14", + "data_header_int16_15", + "data_header_int16_16", + "data_header_int16_17", + "data_header_int16_18", + "data_header_int16_19", + "data_header_int16_20", + "data_header_int16_21", + "data_header_int16_22", + "data_header_int16_23", + "data_header_int16_24", + "data_header_int16_25", + "data_header_int16_26", + "data_header_int16_27", + "data_header_int16_28", + "data_header_int16_29", + "data_header_int16_30", + "data_header_int16_31", + "data_header_int16_32", + "data_header_int16_33", + "data_header_int16_34", + "data_header_int16_35", + "data_header_int16_36", + "data_header_int16_37", + "data_header_int16_38", + "data_header_int16_39", + "data_header_int16_40", + "data_header_int16_41", + "data_header_int16_42", + "data_header_int16_43", + "data_header_int16_44", + "data_header_int16_45", + "data_header_int16_46", + "data_header_int16_47", + "data_header_int16_48", + "data_header_int16_49", + "period_seconds", ) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index a78d846e3d..7338e2a829 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -6,12 +6,14 @@ """Rules for converting NIMROD fields into cubes.""" import warnings +import re import cf_units import cftime import numpy as np import iris +import iris.coord_systems from iris.coords import DimCoord from iris.exceptions import TranslationError @@ -25,12 +27,70 @@ NIMROD_DEFAULT = -32767.0 TIME_UNIT = cf_units.Unit( - "hours since 1970-01-01 00:00:00", calendar=cf_units.CALENDAR_GREGORIAN + "seconds since 1970-01-01 00:00:00", calendar=cf_units.CALENDAR_GREGORIAN ) +DEFAULT_UNITS = {817: 'm s^-1', + 422: 'min^-1', + 218: 'cm', + 155: 'm', + 101: 'm', + 63: 'mm hr^-1', + 61: 'mm', + 58: 'Celsius', + 12: 'mb'} FIELD_CODES = {73: "orography"} -VERTICAL_CODES = {0: "height", 1: "altitude", 12: "levels_below_ground"} +# VERTICAL_CODES contains conversions from the Nimrod Documentation for the +# header entry 20 for the vertical coordinate type +VERTICAL_CODES = {0: {'standard_name': 'height', 'units': 'm', + 'attributes': {"positive": "up"}}, + 1: {'standard_name': 'altitude', 'units': 'm', + 'attributes': {"positive": "up"}}, + 2: {'standard_name': 'pressure', 'units': 'hPa', + 'attributes': {"positive": "down"}}, + 6: {'standard_name': 'temperature', 'units': 'K'}, + 12: {'long_name': 'levels_below_ground', 'units': 'unknown', + 'attributes': {"positive": "down"}}} +# Unhandled VERTICAL_CODES values (no use case identified): +# 3: ['sigma', 'model level'], +# 4: ['eta', 'model level'], +# 5: ['radar beam number', 'unknown'], +# 7: ['potential temperature', 'unknown'], +# 8: ['equivalent potential temperature', 'unknown'], +# 9: ['wet bulb potential temperature', 'unknown'], +# 10: ['potential vorticity', 'unknown'], +# 11: ['cloud boundary', 'unknown'], + +SOIL_TYPE_CODES = {1: "broadleaf_tree", + 2: "needleleaf_tree", + 3: "c3_grass", + 4: "c4_grass", + 5: "crop", + 6: "shrub", + 7: "urban", + 8: "water", + 9: "soil", + 10: "ice", + 601: "urban_canyon", + 602: "urban_roof", + } +TIME_AVERAGING_CODES = { + 8192: "maximum in period", + 4096: 'minimum in period', + 2048: 'unknown(2048)', + 1024: 'unknown(1024)', + 512: 'time lagged', + 256: 'extrapolation', + 128: 'accumulation or average', + 64: 'from UM 150m', + 32: 'scaled to UM resolution', + 16: 'averaged over multiple surface types', + 8: 'only observations used', + 4: 'smoothed', + 2: 'cold bias applied', + 1: 'warm bias applied', +} class TranslationWarning(Warning): @@ -38,7 +98,50 @@ class TranslationWarning(Warning): def name(cube, field): - """Set the cube's name from the field.""" + """Set the cube's name from the field. + Modifies the Nimrod object title based on other meta-data in the + Nimrod field and known use cases. + """ + if field.field_code == 161 and field.threshold_value >= 0.: + field.title = "minimum_cloud_base_above_threshold" + if field.field_code == 12: + field.title = "pressure" + if field.field_code == 28: + field.title = "snow probability" + if field.field_code == 29 and field.threshold_value >= 0.: + field.title = "fog fraction" + if field.field_code == 58: + field.title = "temperature" + if field.field_code == 61: + field.title = "precipitation" + if field.field_code == 63: + field.title = "precipitation" + if field.field_code == 817: + field.title = "wind_speed_of_gust" + if field.field_code == 155: + field.title = "Visibility" + if field.field_code == 218: + field.title = "snowfall" + if field.field_code == 101: + field.title = "snowmelt_above_sea_level" + if field.field_code == 172: + field.title = "cloud_area_fraction_in_atmosphere" + if field.field_code == 421: + field.title = "precipitation type" + if field.field_code == 804 and field.vertical_coord >= 0.: + field.title = "wind speed" + if field.field_code == 806 and field.vertical_coord >= 0.: + field.title = "wind direction" + if field.field_code == 821: + field.title = "Probabilistic Gust Risk Analysis from Observations" + field.source = "Nimrod pwind routine" + if field.source.strip() == "pwind": + field.source = "Nimrod pwind routine" + + if getattr(field, "ensemble_member") == -98 and 'mean' not in field.title: + field.title = 'mean_of_' + field.title + if getattr(field, "ensemble_member") == -99 and 'spread' not in field.title: + field.title = 'standard_deviation_of_' + field.title cube.rename(field.title.strip()) @@ -47,36 +150,105 @@ def units(cube, field): """ Set the cube's units from the field. + Takes into account nimrod unit strings of the form unit*?? where the data + needs to converted by dividing by ??. Also converts units we know Iris + can't handle into appropriate units Iris can handle. This is mostly when + there is an inappropriate capital letter in the unit in the Nimrod file. + Some units still can't be handled by Iris so in these cases empty strings + are added as the cube's unit. The most notable unit Iris can't handle is + oktas for cloud cover. + Unhandled units are stored in an "invalid_units" attribute instead. """ - units = field.units.strip() + unit_exception_dictionary = {'Knts': 'knots', + 'knts': 'knots', + 'J/Kg': 'J/kg', + 'logical': '', + 'Code': '', + 'mask': '', + 'oktas': '', + 'm/2-25k': '', + 'g/Kg': '', + 'unitless': '', + 'Fraction': '1', + 'index': '', + 'Beaufort': '', + 'mmh2o': 'kg/m2', + 'n/a': ''} + + field_units = field.units.strip() + if '*' in field_units: + # Split into unit string and integer + unit_list = field_units.split('*') + if '^' in unit_list[1]: + # Split out magnitude + unit_sublist = unit_list[1].split('^') + cube.data = cube.data / float(unit_sublist[0]) ** float( + unit_sublist[1]) + else: + cube.data = cube.data / float(unit_list[1]) + field_units = unit_list[0] + if 'ug/m3E1' in field_units: + # Split into unit string and integer + unit_list = field_units.split('E') + cube.data = cube.data / 10.**float(unit_list[1]) + field_units = unit_list[0] + if '%' in field_units: + # Convert any percentages into fraction + unit_list = field_units.split('%') + if len(''.join(unit_list)) == 0: + field_units = '1' + cube.data = cube.data / 100. + if field_units in unit_exception_dictionary.keys(): + field_units = unit_exception_dictionary[field_units] + if len(field_units) > 0 and field_units[0] == '/': + # Deal with the case where the units are of the form '/unit' eg + # '/second' in the Nimrod file. This converts to the form unit^-1 + field_units = field_units[1:] + '^-1' try: - cube.units = units + cube.units = field_units except ValueError: # Just add it as an attribute. warnings.warn( - "Unhandled units '{0}' recorded in cube attributes.".format(units) + "Unhandled units '{0}' recorded in cube attributes.".format( + field_units) ) - cube.attributes["invalid_units"] = units + cube.attributes["invalid_units"] = field_units def time(cube, field): """Add a time coord to the cube.""" - valid_date = cftime.datetime( - field.vt_year, - field.vt_month, - field.vt_day, - field.vt_hour, - field.vt_minute, - field.vt_second, - ) + if field.vt_year <= 0: + # Some ancillary files, eg land sea mask do not + # have a validity time. So make one up for the + # start of the year. + # This will screw up the forecast_period for these fields, + # although if the valid time is missing too, it will be + # made to be the same, so the forecast_period will always + # be zero for these files. + valid_date = cftime.datetime( + 2016, 1, 1, 0, 0, 0) + else: + valid_date = cftime.datetime( + field.vt_year, + field.vt_month, + field.vt_day, + field.vt_hour, + field.vt_minute, + field.vt_second, + ) point = TIME_UNIT.date2num(valid_date) - bounds = None - if field.period_minutes != field.int_mdi and field.period_minutes != 0: - # Create a bound array to handle the Period of Interest if set. - bounds = (point - (field.period_minutes / 60.0), point) + lb_delta = None + if field.period_minutes == 32767: + lb_delta = int(field.period_seconds) + elif field.period_minutes != field.int_mdi and field.period_minutes != 0: + lb_delta = int(field.period_minutes) * 60 + if lb_delta: + bounds = [point - lb_delta, point] + else: + bounds = None time_coord = DimCoord( points=point, bounds=bounds, standard_name="time", units=TIME_UNIT @@ -87,7 +259,7 @@ def time(cube, field): def reference_time(cube, field): """Add a 'reference time' to the cube, if present in the field.""" - if field.dt_year != field.int_mdi: + if field.dt_year != field.int_mdi and field.dt_year > 0: data_date = cftime.datetime( field.dt_year, field.dt_month, @@ -105,6 +277,26 @@ def reference_time(cube, field): cube.add_aux_coord(ref_time_coord) +def mask_cube(cube, field): + """ + Updates cube.data to be a masked array if appropriate. + + """ + + if field.datum_type == 1: + # field.data are integers + if np.any(field.data == field.int_mdi): + cube.data = np.ma.masked_equal(field.data, + field.int_mdi) + elif field.datum_type == 0: + # field.data are floats + if np.any(np.isclose(field.data, field.float32_mdi)): + cube.data = np.ma.masked_inside( + field.data, + field.float32_mdi - 0.5, + field.float32_mdi + 0.5) + + def experiment(cube, field): """Add an 'experiment number' to the cube, if present in the field.""" if field.experiment_num != field.int_mdi: @@ -113,170 +305,171 @@ def experiment(cube, field): ) -def proj_biaxial_ellipsoid(cube, field): +def proj_biaxial_ellipsoid(field): """ - Ellipsoid definition is currently ignored. - - """ - pass + Returns the correct dictionary of arguments needed to define an + iris.coord_systems.GeogCS. + Based firstly on the value given by ellipsoid, then by grid if ellipsoid is + missing, select the right pre-defined ellipsoid dictionary (Airy_1830 or + International_1924). -def tm_meridian_scaling(cube, field): """ - Deal with the scale factor on the central meridian for transverse mercator - projections if present in the field. - - Currently only caters for British National Grid. - - """ - if field.tm_meridian_scaling not in [field.float32_mdi, NIMROD_DEFAULT]: - if abs(field.tm_meridian_scaling - MERIDIAN_SCALING_BNG) < 1e-6: - pass # This is the expected value for British National Grid + # Reference for Airy_1830 and international_1924 ellipsoids: + # http://fcm9/projects/PostProc/wiki/PostProcDocDomains#ProjectionConstants + # Reference for GRS80: + Airy_1830 = {'semi_major_axis': 6377563.396, 'semi_minor_axis': 6356256.910} + International_1924 = {'semi_major_axis': 6378388.000, + 'semi_minor_axis': 6356911.946} + if field.proj_biaxial_ellipsoid == 0: + ellipsoid = Airy_1830 + elif field.proj_biaxial_ellipsoid == 1: + ellipsoid = International_1924 + elif field.proj_biaxial_ellipsoid == field.int_mdi or \ + field.proj_biaxial_ellipsoid == -32767: + if field.horizontal_grid_type == 0: + ellipsoid = Airy_1830 + elif field.horizontal_grid_type == 1 or field.horizontal_grid_type == 4: + ellipsoid = International_1924 else: - warnings.warn( - "tm_meridian_scaling not yet handled: {}" - "".format(field.tm_meridian_scaling), - TranslationWarning, - ) - - -def british_national_grid_x(cube, field): - """Add a British National Grid X coord to the cube.""" - x_coord = DimCoord( - np.arange(field.num_cols) * field.column_step + field.x_origin, - standard_name="projection_x_coordinate", - units="m", - coord_system=iris.coord_systems.OSGB(), - ) - cube.add_dim_coord(x_coord, 1) - - -def british_national_grid_y(cube, field): - """ - Add a British National Grid Y coord to the cube. - - Currently only handles origin in the top left corner. - - """ - if field.origin_corner == 0: # top left - y_coord = DimCoord( - np.arange(field.num_rows)[::-1] * -field.row_step + field.y_origin, - standard_name="projection_y_coordinate", - units="m", - coord_system=iris.coord_systems.OSGB(), - ) - cube.add_dim_coord(y_coord, 0) + raise TranslationError('''Unsupported grid type, only NG, EuroPP + and lat/long are possible''') else: raise TranslationError( - "Corner {0} not yet implemented".format(field.origin_corner) - ) - - -def horizontal_grid(cube, field): - """Add X and Y coords to the cube. - - Currently only handles British National Grid. + 'Ellipsoid not supported, proj_biaxial_ellipsoid:{}, ' + 'horizontal_grid_type:{}'.format(field.proj_biaxial_ellipsoid, + field.horizontal_grid_type)) + return ellipsoid + + +def set_british_national_grid_defaults(field): + """Check for missing coord-system meta-data and set default values for + the Ordnance Survey GB Transverse Mercator projection. Some Radarnet + files are missing these.""" + invalid_values = (-32767., float(field.int_mdi)) + + if np.any([field.true_origin_latitude == v for v in invalid_values]): + field.true_origin_latitude = 49. + if np.any([field.true_origin_longitude == v for v in invalid_values]): + field.true_origin_longitude = -2. + if np.any([field.true_origin_easting == v for v in invalid_values]): + field.true_origin_easting = 400000. + if np.any([field.true_origin_northing == v for v in invalid_values]): + field.true_origin_northing = -100000. + if np.any([field.tm_meridian_scaling == v for v in invalid_values]): + field.tm_meridian_scaling = 0.9996012717 + + ng_central_meridian_sf_dp = 0.9996012717 + if abs(field.tm_meridian_scaling - ng_central_meridian_sf_dp) < 1.0e-04: + # Update the National Grid scaling factor to double + # precision accuracy to improve the accuracy of + # reprojection calculations that use it. + field.tm_meridian_scaling = ng_central_meridian_sf_dp + + +def coord_system(field): + """Define the coordinate system for the field.""" + ellipsoid = proj_biaxial_ellipsoid(field) - """ - # "NG" (British National Grid) if field.horizontal_grid_type == 0: - british_national_grid_x(cube, field) - british_national_grid_y(cube, field) + # Check for missing grid meta-data and insert OSGB definitions. + # Some Radarnet files are missing these. + set_british_national_grid_defaults(field) + if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: + coord_sys = iris.coord_systems.TransverseMercator( + field.true_origin_latitude, field.true_origin_longitude, field.true_origin_easting, + field.true_origin_northing, + field.tm_meridian_scaling, iris.coord_systems.GeogCS(**ellipsoid)) + elif field.horizontal_grid_type == 1: + coord_sys = iris.coord_systems.GeogCS(**ellipsoid) else: raise TranslationError( - "Grid type %d not yet implemented" % field.horizontal_grid_type + "Coordinate system for field type {} not implemented".format( + field.horizontal_grid_type + ) ) + return coord_sys -def orography_vertical_coord(cube, field): - """Special handling of vertical coords for orography fields: Do nothing.""" - # We can find values in the vertical coord, such as 9999, - # for orography fields. Don't make a vertical coord from these. - pass - +def horizontal_grid(cube, field): + """Add X and Y coordinates to the cube. -def height_vertical_coord(cube, field): - """Add a height coord to the cube, if present in the field.""" - if ( - field.reference_vertical_coord_type == field.int_mdi - or field.reference_vertical_coord == field.float32_mdi - ): - height_coord = DimCoord( - field.vertical_coord, - standard_name="height", - units="m", - attributes={"positive": "up"}, - ) - cube.add_aux_coord(height_coord) - else: - raise TranslationError("Bounded vertical not yet implemented") - - -def altitude_vertical_coord(cube, field): - """Add an altitude coord to the cube, if present in the field.""" - if ( - field.reference_vertical_coord_type == field.int_mdi - or field.reference_vertical_coord == field.float32_mdi - ): - alti_coord = DimCoord( - field.vertical_coord, - standard_name="altitude", - units="m", - attributes={"positive": "up"}, - ) - cube.add_aux_coord(alti_coord) - else: - raise TranslationError("Bounded vertical not yet implemented") - - -def levels_below_ground_vertical_coord(cube, field): - """Add a levels_below_ground coord to the cube, if present in the field.""" - if ( - field.reference_vertical_coord_type == field.int_mdi - or field.reference_vertical_coord == field.float32_mdi - ): - lev_coord = DimCoord( - field.vertical_coord, - long_name="levels_below_ground", - units="1", - attributes={"positive": "down"}, + """ + if field.origin_corner != 0: + raise TranslationError( + "Corner {0} not yet implemented".format(field.origin_corner) ) - cube.add_aux_coord(lev_coord) + crs = coord_system(field) + if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: + units_name = 'm' + x_coord_name = 'projection_x_coordinate' + y_coord_name = 'projection_y_coordinate' + elif field.horizontal_grid_type == 1: + units_name = 'degrees' + x_coord_name = 'longitude' + y_coord_name = 'latitude' else: - raise TranslationError("Bounded vertical not yet implemented") + raise TranslationError("Horizontal grid type {} not " + "implemented".format(field.horizontal_grid_type)) + x_coord = DimCoord( + np.arange(field.num_cols) * field.column_step + field.x_origin, + standard_name=x_coord_name, + units=units_name, + coord_system=crs, + ) + cube.add_dim_coord(x_coord, 1) + y_coord = DimCoord( + np.arange(field.num_rows)[::-1] * -field.row_step + field.y_origin, + standard_name=y_coord_name, + units=units_name, + coord_system=crs, + ) + cube.add_dim_coord(y_coord, 0) def vertical_coord(cube, field): - """Add a vertical coord to the cube.""" - v_type = field.vertical_coord_type - - if v_type not in [field.int_mdi, NIMROD_DEFAULT]: - if FIELD_CODES.get(field.field_code, None) == "orography": - orography_vertical_coord(cube, field) - else: - vertical_code_name = VERTICAL_CODES.get(v_type, None) - if vertical_code_name == "height": - height_vertical_coord(cube, field) - elif vertical_code_name == "altitude": - altitude_vertical_coord(cube, field) - elif vertical_code_name == "levels_below_ground": - levels_below_ground_vertical_coord(cube, field) - else: - warnings.warn( - "Vertical coord {!r} not yet handled" "".format(v_type), - TranslationWarning, - ) + """Add a vertical coord to the cube, if appropriate.""" + coord_point = getattr(field, "vertical_coord") + if (coord_point == 9999. and + (field.reference_vertical_coord in [9999., field.int_mdi, + NIMROD_DEFAULT])): + return + if coord_point in [9999., 8888.]: + # A bounded vertical coord starting from the surface + # Relies on correct field.vertical_coord_type to distinguish between + # these meanings. + coord_point = 0. + + if (field.reference_vertical_coord >= 0. and + field.reference_vertical_coord != coord_point): + bounds = [coord_point, field.reference_vertical_coord] + else: + bounds = None + + coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) + if coord_args: + new_coord = iris.coords.AuxCoord(coord_point, bounds=bounds, + **coord_args) + # Add coordinate to cube + cube.add_aux_coord(new_coord) + return + + warnings.warn( + "Vertical coord {!r} not yet handled" "".format( + field.vertical_coord_type), + TranslationWarning, + ) def ensemble_member(cube, field): """Add an 'ensemble member' coord to the cube, if present in the field.""" - ensemble_member = getattr(field, "ensemble_member") - if ensemble_member != field.int_mdi: - cube.add_aux_coord(DimCoord(ensemble_member, "realization")) + ensemble_member_value = getattr(field, "ensemble_member") + if ensemble_member_value != field.int_mdi: + cube.add_aux_coord(DimCoord(ensemble_member_value, "realization")) def origin_corner(cube, field): - """Ensure the data matches the order of the coords we've made.""" + """Ensure the data matches the order of the coordinates we've made.""" if field.origin_corner == 0: # top left cube.data = cube.data[::-1, :].copy() else: @@ -289,12 +482,12 @@ def origin_corner(cube, field): def attributes(cube, field): """Add attributes to the cube.""" - def add_attr(name): + def add_attr(item): """Add an attribute to the cube.""" - if hasattr(field, name): - value = getattr(field, name) + if hasattr(field, item): + value = getattr(field, item) if value not in [field.int_mdi, field.float32_mdi]: - cube.attributes[name] = value + cube.attributes[item] = value add_attr("nimrod_version") add_attr("field_code") @@ -317,8 +510,118 @@ def add_attr(name): add_attr("sensor_id") add_attr("meteosat_id") add_attr("alphas_available") + for key in ["neighbourhood_radius", "recursive_filter_iterations", + "recursive_filter_alpha", "threshold_vicinity_radius", + "probability_period_of_event"]: + add_attr(key) - cube.attributes["source"] = field.source.strip() + source = field.source.strip() + rematcher = re.compile('^ek\d\d$') + if (rematcher.match(source) is not None + or source.find('umek') == 0): + source = 'MOGREPS-UK' + cube.attributes['source'] = source + + +def threshold_coord(cube, field): + """ + Adds a scalar threshold coord to the cube for known use cases. + """ + coord_keys = None + if field.field_code == 161 and field.threshold_value >= 0.: + coord_keys = {"standard_name": "cloud_area_fraction", + "units": ""} + if field.field_code == 29 and field.threshold_value >= 0.: + coord_keys = {"standard_name": "visibility_in_air", + "units": "metres"} + if field.field_code == 821: + coord_keys = {"standard_name": "wind_speed_of_gust", + "units": "m/s"} + if coord_keys: + cube.add_aux_coord(iris.coords.AuxCoord(field.threshold_value, + var_name='threshold', + **coord_keys)) + + +def probability_coord(cube, field): + """ + Adds a coord relating to probability meta-data from the header to the + cube if appropriate. + Returns True if this is a blended multi-member field + """ + probtype_lookup = {1: {'var_name': 'threshold', + 'attributes': {'relative_to_threshold': 'above'}}, + 2: {'var_name': 'threshold', + 'attributes': {'relative_to_threshold': 'below'}}, + 3: {'standard_name': 'percentile', 'units': "%"}, + 4: {'var_name': 'threshold', + 'attributes': {'relative_to_threshold': 'equal'}}} + probmethod_lookup = {1: 'AOT (Any One Time)', + 2: 'ST (Some Time)', + 4: 'AT (All Time)', + 8: 'AOL (Any One Location)', + 16: 'SW (Some Where)'} + is_multi_member_field = False + coord_keys = probtype_lookup.get(field.threshold_type, None) + if not coord_keys: + return is_multi_member_field + if not coord_keys.get('units', None): + coord_keys['units'] = DEFAULT_UNITS.get(field.field_code, None) + coord_val = None + if field.threshold_value_alt > -32766.: + coord_val = field.threshold_value_alt + elif field.threshold_value > -32766.: + coord_val = field.threshold_value + if field.chead.find('pc') > 0: + try: + coord_val = [int(x.strip('pc')) for x in field.chead.split(' ') + if x.find('pc') > 0][0] + except IndexError: + pass + if coord_val is not None: + if field.threshold_fuzziness > -32766.: + bounds = [coord_val * field.threshold_fuzziness, + coord_val * (2. - field.threshold_fuzziness)] + else: + bounds = None + new_coord = iris.coords.AuxCoord(coord_val, bounds=bounds, **coord_keys) + cube.add_aux_coord(new_coord) + cube.units = '1' + is_multi_member_field = True + + if field.probability_method > 0: + probability_attributes = [] + num = field.probability_method + for key in sorted(probmethod_lookup.keys(), reverse=True): + if num >= key: + probability_attributes.append(probmethod_lookup[key]) + num = num - key + cube.attributes['Probability methods'] = probability_attributes + if field.member_count == 1: + is_multi_member_field = False + return is_multi_member_field + + +def soil_type_coord(cube, field): + """Add soil type as a coord if appropriate""" + if field.threshold_type != 0: + soil_name = SOIL_TYPE_CODES.get(field.soil_type, None) + if soil_name: + cube.add_aux_coord(iris.coords.AuxCoord( + soil_name, + standard_name='soil_type', units=None)) + + +def time_averaging(cube, field): + """Decode the averagingtype code - similar to the PP LBPROC code.""" + num = field.averagingtype + averaging_attributes = [] + for key in sorted(TIME_AVERAGING_CODES.keys(), reverse=True): + if num >= key: + averaging_attributes.append(TIME_AVERAGING_CODES[key]) + num = num - key + if averaging_attributes: + cube.attributes['processing'] = averaging_attributes def run(field): @@ -339,6 +642,8 @@ def run(field): name(cube, field) units(cube, field) + mask_cube(cube, field) + # time time(cube, field) reference_time(cube, field) @@ -346,17 +651,21 @@ def run(field): experiment(cube, field) # horizontal grid - proj_biaxial_ellipsoid(cube, field) - tm_meridian_scaling(cube, field) horizontal_grid(cube, field) # vertical vertical_coord(cube, field) # add other stuff, if present - ensemble_member(cube, field) + soil_type_coord(cube, field) + threshold_coord(cube, field) + if not probability_coord(cube, field): + ensemble_member(cube, field) + time_averaging(cube, field) attributes(cube, field) origin_corner(cube, field) + + cube.data = cube.data.astype(np.float32) return cube diff --git a/lib/iris/tests/integration/test_regridding.py b/lib/iris/tests/integration/test_regridding.py index f16b7f4ab5..ae24f5a10b 100644 --- a/lib/iris/tests/integration/test_regridding.py +++ b/lib/iris/tests/integration/test_regridding.py @@ -59,11 +59,11 @@ def _regrid(self, method): def test_linear(self): res = self._regrid("linear") - self.assertArrayShapeStats(res, (73, 96), -16100.351951, 5603.850769) + self.assertArrayShapeStats(res, (73, 96), -15539.099752, 5603.850769) def test_nearest(self): res = self._regrid("nearest") - self.assertArrayShapeStats(res, (73, 96), -16095.965585, 5612.657155) + self.assertArrayShapeStats(res, (73, 96), -15542.226501, 5612.657155) @tests.skip_data diff --git a/lib/iris/tests/results/nimrod/load_2flds.cml b/lib/iris/tests/results/nimrod/load_2flds.cml index d57761e235..9616021411 100644 --- a/lib/iris/tests/results/nimrod/load_2flds.cml +++ b/lib/iris/tests/results/nimrod/load_2flds.cml @@ -1,41 +1,33 @@ - + - - - - - - - - - + - - + + - - + - + - + diff --git a/lib/iris/tests/results/nimrod/period_of_interest.cml b/lib/iris/tests/results/nimrod/period_of_interest.cml index ef0f33649e..6b673aa3be 100644 --- a/lib/iris/tests/results/nimrod/period_of_interest.cml +++ b/lib/iris/tests/results/nimrod/period_of_interest.cml @@ -3,7 +3,8 @@ - + diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 14295f4079..c849543d2b 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -55,8 +55,6 @@ def test_orography(self): nimrod_load_rules.name(cube, field) nimrod_load_rules.units(cube, field) nimrod_load_rules.reference_time(cube, field) - nimrod_load_rules.proj_biaxial_ellipsoid(cube, field) - nimrod_load_rules.tm_meridian_scaling(cube, field) nimrod_load_rules.vertical_coord(cube, field) nimrod_load_rules.attributes(cube, field) diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index cefea984bb..d189aa7b02 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -46,13 +46,6 @@ def test_unhandled(self): "Vertical coord -1 not yet handled", TranslationWarning ) - def test_orography(self): - name = "orography_vertical_coord" - with mock.patch(self.NIMROD_LOCATION + "." + name) as orog: - self.field.field_code = 73 - self._call_vertical_coord(None) - orog.assert_called_once_with(self.cube, self.field) - def test_height(self): name = "height_vertical_coord" with mock.patch(self.NIMROD_LOCATION + "." + name) as height: From c7cac57d8ba5654dafcd15e35d0056c9964a7ce0 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 27 Jan 2020 12:12:21 +0000 Subject: [PATCH 02/46] Updates unit-test results --- lib/iris/tests/integration/test_regridding.py | 4 ++-- lib/iris/tests/results/nimrod/load_2flds.cml | 4 ++-- lib/iris/tests/results/nimrod/period_of_interest.cml | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/iris/tests/integration/test_regridding.py b/lib/iris/tests/integration/test_regridding.py index ae24f5a10b..0a6a78f2df 100644 --- a/lib/iris/tests/integration/test_regridding.py +++ b/lib/iris/tests/integration/test_regridding.py @@ -59,11 +59,11 @@ def _regrid(self, method): def test_linear(self): res = self._regrid("linear") - self.assertArrayShapeStats(res, (73, 96), -15539.099752, 5603.850769) + self.assertArrayShapeStats(res, (73, 96), -15539.099752, 5825.651452) def test_nearest(self): res = self._regrid("nearest") - self.assertArrayShapeStats(res, (73, 96), -15542.226501, 5612.657155) + self.assertArrayShapeStats(res, (73, 96), -15542.226501, 5839.922010) @tests.skip_data diff --git a/lib/iris/tests/results/nimrod/load_2flds.cml b/lib/iris/tests/results/nimrod/load_2flds.cml index 9616021411..835e10d751 100644 --- a/lib/iris/tests/results/nimrod/load_2flds.cml +++ b/lib/iris/tests/results/nimrod/load_2flds.cml @@ -9,11 +9,11 @@ - + + ..., 851999.984375, 853999.984375, 855999.984375]" shape="(548,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64"> diff --git a/lib/iris/tests/results/nimrod/period_of_interest.cml b/lib/iris/tests/results/nimrod/period_of_interest.cml index 6b673aa3be..b71202f56e 100644 --- a/lib/iris/tests/results/nimrod/period_of_interest.cml +++ b/lib/iris/tests/results/nimrod/period_of_interest.cml @@ -3,8 +3,7 @@ - + From d7e6b1de1faf800dd436a70250b75d91be2aad6a Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 27 Jan 2020 14:38:20 +0000 Subject: [PATCH 03/46] Fixes precision of data and coords. Updates existing unit-tests. --- lib/iris/fileformats/nimrod_load_rules.py | 37 ++++++++++++------- lib/iris/tests/results/nimrod/load_2flds.cml | 12 +++--- .../results/nimrod/period_of_interest.cml | 2 +- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 7338e2a829..6ae4e7a7fc 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -238,7 +238,7 @@ def time(cube, field): field.vt_minute, field.vt_second, ) - point = TIME_UNIT.date2num(valid_date) + point = np.array(TIME_UNIT.date2num(valid_date), dtype=np.int64) lb_delta = None if field.period_minutes == 32767: @@ -246,7 +246,7 @@ def time(cube, field): elif field.period_minutes != field.int_mdi and field.period_minutes != 0: lb_delta = int(field.period_minutes) * 60 if lb_delta: - bounds = [point - lb_delta, point] + bounds = np.array([point - lb_delta, point], dtype=np.int64) else: bounds = None @@ -269,7 +269,7 @@ def reference_time(cube, field): ) ref_time_coord = DimCoord( - TIME_UNIT.date2num(data_date), + np.array(TIME_UNIT.date2num(data_date), dtype=np.int64), standard_name="forecast_reference_time", units=TIME_UNIT, ) @@ -411,15 +411,19 @@ def horizontal_grid(cube, field): else: raise TranslationError("Horizontal grid type {} not " "implemented".format(field.horizontal_grid_type)) + points = (np.arange(field.num_cols) * field.column_step + + field.x_origin).astype(np.float32) x_coord = DimCoord( - np.arange(field.num_cols) * field.column_step + field.x_origin, + points, standard_name=x_coord_name, units=units_name, coord_system=crs, ) cube.add_dim_coord(x_coord, 1) + points = (np.arange(field.num_rows)[::-1] * -field.row_step + + field.y_origin).astype(np.float32) y_coord = DimCoord( - np.arange(field.num_rows)[::-1] * -field.row_step + field.y_origin, + points, standard_name=y_coord_name, units=units_name, coord_system=crs, @@ -439,10 +443,11 @@ def vertical_coord(cube, field): # Relies on correct field.vertical_coord_type to distinguish between # these meanings. coord_point = 0. - + coord_point = np.array(coord_point, dtype=np.float32) if (field.reference_vertical_coord >= 0. and field.reference_vertical_coord != coord_point): - bounds = [coord_point, field.reference_vertical_coord] + bounds = np.array([coord_point, field.reference_vertical_coord], + dtype=np.float32) else: bounds = None @@ -465,7 +470,9 @@ def ensemble_member(cube, field): """Add an 'ensemble member' coord to the cube, if present in the field.""" ensemble_member_value = getattr(field, "ensemble_member") if ensemble_member_value != field.int_mdi: - cube.add_aux_coord(DimCoord(ensemble_member_value, "realization")) + cube.add_aux_coord(DimCoord(np.array(ensemble_member_value, + dtype=np.int32), + "realization")) def origin_corner(cube, field): @@ -538,9 +545,10 @@ def threshold_coord(cube, field): coord_keys = {"standard_name": "wind_speed_of_gust", "units": "m/s"} if coord_keys: - cube.add_aux_coord(iris.coords.AuxCoord(field.threshold_value, - var_name='threshold', - **coord_keys)) + cube.add_aux_coord(iris.coords.AuxCoord( + np.array(field.threshold_value, dtype=np.float32), + var_name='threshold', + **coord_keys)) def probability_coord(cube, field): @@ -584,7 +592,10 @@ def probability_coord(cube, field): coord_val * (2. - field.threshold_fuzziness)] else: bounds = None - new_coord = iris.coords.AuxCoord(coord_val, bounds=bounds, **coord_keys) + new_coord = iris.coords.AuxCoord( + np.array(coord_val, dtype=np.float32), + bounds=np.array(bounds, dtype=np.float32), + **coord_keys) cube.add_aux_coord(new_coord) cube.units = '1' is_multi_member_field = True @@ -637,7 +648,7 @@ def run(field): * A new :class:`~iris.cube.Cube`, created from the NimrodField. """ - cube = iris.cube.Cube(field.data) + cube = iris.cube.Cube(field.data.astype(np.float32)) name(cube, field) units(cube, field) diff --git a/lib/iris/tests/results/nimrod/load_2flds.cml b/lib/iris/tests/results/nimrod/load_2flds.cml index 835e10d751..9eeb754d66 100644 --- a/lib/iris/tests/results/nimrod/load_2flds.cml +++ b/lib/iris/tests/results/nimrod/load_2flds.cml @@ -9,22 +9,22 @@ - + - + - + - + diff --git a/lib/iris/tests/results/nimrod/period_of_interest.cml b/lib/iris/tests/results/nimrod/period_of_interest.cml index b71202f56e..258e5bcbbc 100644 --- a/lib/iris/tests/results/nimrod/period_of_interest.cml +++ b/lib/iris/tests/results/nimrod/period_of_interest.cml @@ -3,7 +3,7 @@ - + From a428a0b5d49c0f13f2a1910ed1a7e4a6fee5f710 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 27 Jan 2020 16:33:58 +0000 Subject: [PATCH 04/46] Updates existing unit-tests. --- .../test_tm_meridian_scaling.py | 59 ------------------- .../nimrod_load_rules/test_vertical_coord.py | 5 +- 2 files changed, 4 insertions(+), 60 deletions(-) delete mode 100644 lib/iris/tests/unit/fileformats/nimrod_load_rules/test_tm_meridian_scaling.py diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_tm_meridian_scaling.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_tm_meridian_scaling.py deleted file mode 100644 index 39f08edc7d..0000000000 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_tm_meridian_scaling.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -Unit tests for the `iris.fileformats.nimrod_load_rules.tm_meridian_scaling` -function. - -""" - -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests - -from unittest import mock - -from iris.fileformats.nimrod_load_rules import ( - tm_meridian_scaling, - NIMROD_DEFAULT, - MERIDIAN_SCALING_BNG, -) -from iris.fileformats.nimrod import NimrodField - - -class Test(tests.IrisTest): - def setUp(self): - self.field = mock.Mock( - tm_meridian_scaling=NIMROD_DEFAULT, - spec=NimrodField, - float32_mdi=-123, - ) - self.cube = mock.Mock() - - def _call_tm_meridian_scaling(self, scaling_value): - self.field.tm_meridian_scaling = scaling_value - tm_meridian_scaling(self.cube, self.field) - - def test_unhandled(self): - with mock.patch("warnings.warn") as warn: - self._call_tm_meridian_scaling(1) - self.assertEqual(warn.call_count, 1) - - @tests.no_warnings - def test_british_national_grid(self): - # A value is not returned in this rule currently. - self.assertEqual( - None, self._call_tm_meridian_scaling(MERIDIAN_SCALING_BNG) - ) - - def test_null(self): - with mock.patch("warnings.warn") as warn: - self._call_tm_meridian_scaling(NIMROD_DEFAULT) - self._call_tm_meridian_scaling(self.field.float32_mdi) - self.assertEqual(warn.call_count, 0) - - -if __name__ == "__main__": - tests.main() diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index d189aa7b02..dc666c50f8 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -31,6 +31,9 @@ def setUp(self): vertical_coord_type=NIMROD_DEFAULT, int_mdi=mock.sentinel.int_mdi, field_code=mock.sentinel.field_code, + vertical_coord=mock.sentinel.vertical_coord, + reference_vertical_coord=mock.sentinel.reference_vertical_coord, + ensemble_member=NIMROD_DEFAULT, spec=NimrodField, ) self.cube = mock.Mock() @@ -47,7 +50,7 @@ def test_unhandled(self): ) def test_height(self): - name = "height_vertical_coord" + name = "vertical_coord" with mock.patch(self.NIMROD_LOCATION + "." + name) as height: self._call_vertical_coord(0) height.assert_called_once_with(self.cube, self.field) From f16fb3a4fe4e9b750d1278de90f71a5f6ff51108 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 28 Jan 2020 10:10:35 +0000 Subject: [PATCH 05/46] Removes trailing whitespace. --- lib/iris/fileformats/nimrod_load_rules.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 6ae4e7a7fc..8101098ce4 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -279,7 +279,7 @@ def reference_time(cube, field): def mask_cube(cube, field): """ - Updates cube.data to be a masked array if appropriate. + Updates cube.data to be a masked array if appropriate. """ @@ -440,7 +440,7 @@ def vertical_coord(cube, field): return if coord_point in [9999., 8888.]: # A bounded vertical coord starting from the surface - # Relies on correct field.vertical_coord_type to distinguish between + # Relies on correct field.vertical_coord_type to distinguish between # these meanings. coord_point = 0. coord_point = np.array(coord_point, dtype=np.float32) @@ -450,7 +450,7 @@ def vertical_coord(cube, field): dtype=np.float32) else: bounds = None - + coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) if coord_args: new_coord = iris.coords.AuxCoord(coord_point, bounds=bounds, @@ -458,7 +458,7 @@ def vertical_coord(cube, field): # Add coordinate to cube cube.add_aux_coord(new_coord) return - + warnings.warn( "Vertical coord {!r} not yet handled" "".format( field.vertical_coord_type), @@ -517,8 +517,8 @@ def add_attr(item): add_attr("sensor_id") add_attr("meteosat_id") add_attr("alphas_available") - for key in ["neighbourhood_radius", "recursive_filter_iterations", - "recursive_filter_alpha", "threshold_vicinity_radius", + for key in ["neighbourhood_radius", "recursive_filter_iterations", + "recursive_filter_alpha", "threshold_vicinity_radius", "probability_period_of_event"]: add_attr(key) @@ -549,11 +549,11 @@ def threshold_coord(cube, field): np.array(field.threshold_value, dtype=np.float32), var_name='threshold', **coord_keys)) - - + + def probability_coord(cube, field): """ - Adds a coord relating to probability meta-data from the header to the + Adds a coord relating to probability meta-data from the header to the cube if appropriate. Returns True if this is a blended multi-member field """ @@ -582,7 +582,7 @@ def probability_coord(cube, field): coord_val = field.threshold_value if field.chead.find('pc') > 0: try: - coord_val = [int(x.strip('pc')) for x in field.chead.split(' ') + coord_val = [int(x.strip('pc')) for x in field.chead.split(' ') if x.find('pc') > 0][0] except IndexError: pass @@ -676,7 +676,7 @@ def run(field): attributes(cube, field) origin_corner(cube, field) - + cube.data = cube.data.astype(np.float32) return cube From 22f2fcc8161851a7a9da8e530a124b36e71a7883 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 30 Jan 2020 15:17:59 +0000 Subject: [PATCH 06/46] Improves accuracy and precision of meta-data handling and known exceptions. Adds forecast_period coord. Adds more Nimrod data to unit-tests to cover known operational data. --- lib/iris/fileformats/nimrod_load_rules.py | 225 ++- .../results/nimrod/levels_below_ground.cml | 4 +- lib/iris/tests/results/nimrod/load_2flds.cml | 7 +- lib/iris/tests/results/nimrod/mockography.cml | 2 +- .../results/nimrod/probability_fields.cml | 1792 +++++++++++++++++ .../nimrod/u1096_ng_bmr04_precip_2km.cml | 43 + .../u1096_ng_bsr05_precip_accum60_2km.cml | 41 + .../nimrod/u1096_ng_ek00_cloud3d0060_2km.cml | 105 + .../nimrod/u1096_ng_ek00_cloud_2km.cml | 279 +++ .../nimrod/u1096_ng_ek00_convection_2km.cml | 361 ++++ .../nimrod/u1096_ng_ek00_convwind_2km.cml | 267 +++ .../nimrod/u1096_ng_ek00_frzlev_2km.cml | 303 +++ .../nimrod/u1096_ng_ek00_height_2km.cml | 40 + .../nimrod/u1096_ng_ek00_precip_2km.cml | 116 ++ .../nimrod/u1096_ng_ek00_precipaccum_2km.cml | 43 + .../nimrod/u1096_ng_ek00_preciptype_2km.cml | 342 ++++ .../nimrod/u1096_ng_ek00_pressure_2km.cml | 77 + .../nimrod/u1096_ng_ek00_radiation_2km.cml | 262 +++ .../nimrod/u1096_ng_ek00_radiationuv_2km.cml | 114 ++ .../results/nimrod/u1096_ng_ek00_refl_2km.cml | 49 + .../u1096_ng_ek00_relhumidity3d0060_2km.cml | 57 + .../nimrod/u1096_ng_ek00_relhumidity_2km.cml | 47 + .../results/nimrod/u1096_ng_ek00_snow_2km.cml | 115 ++ .../nimrod/u1096_ng_ek00_soil3d0060_2km.cml | 179 ++ .../results/nimrod/u1096_ng_ek00_soil_2km.cml | 124 ++ .../nimrod/u1096_ng_ek00_temperature_2km.cml | 181 ++ .../nimrod/u1096_ng_ek00_visibility_2km.cml | 223 ++ .../results/nimrod/u1096_ng_ek00_wind_2km.cml | 269 +++ .../nimrod/u1096_ng_ek00_winduv3d0015_2km.cml | 91 + .../nimrod/u1096_ng_ek00_winduv_2km.cml | 91 + .../results/nimrod/u1096_ng_ek01_cape_2km.cml | 228 +++ ...u1096_ng_ek07_precip0540_accum180_18km.cml | 41 + .../results/nimrod/u1096_ng_umqv_fog_2km.cml | 48 + lib/iris/tests/test_nimrod.py | 49 + .../nimrod_load_rules/test_vertical_coord.py | 38 +- 35 files changed, 6187 insertions(+), 66 deletions(-) create mode 100644 lib/iris/tests/results/nimrod/probability_fields.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml create mode 100644 lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 8101098ce4..2e704e3354 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -15,7 +15,7 @@ import iris import iris.coord_systems from iris.coords import DimCoord -from iris.exceptions import TranslationError +from iris.exceptions import TranslationError, CoordinateNotFoundError __all__ = ["run"] @@ -32,14 +32,17 @@ DEFAULT_UNITS = {817: 'm s^-1', + 804: 'knots', 422: 'min^-1', 218: 'cm', + 172: 'oktas', 155: 'm', 101: 'm', 63: 'mm hr^-1', 61: 'mm', 58: 'Celsius', - 12: 'mb'} + 12: 'mb', + 8: '1'} FIELD_CODES = {73: "orography"} # VERTICAL_CODES contains conversions from the Nimrod Documentation for the # header entry 20 for the vertical coordinate type @@ -47,10 +50,10 @@ 'attributes': {"positive": "up"}}, 1: {'standard_name': 'altitude', 'units': 'm', 'attributes': {"positive": "up"}}, - 2: {'standard_name': 'pressure', 'units': 'hPa', + 2: {'standard_name': 'air_pressure', 'units': 'hPa', 'attributes': {"positive": "down"}}, - 6: {'standard_name': 'temperature', 'units': 'K'}, - 12: {'long_name': 'levels_below_ground', 'units': 'unknown', + 6: {'standard_name': 'air_temperature', 'units': 'K'}, + 12: {'long_name': 'depth_below_ground', 'units': 'm', 'attributes': {"positive": "down"}}} # Unhandled VERTICAL_CODES values (no use case identified): # 3: ['sigma', 'model level'], @@ -102,10 +105,10 @@ def name(cube, field): Modifies the Nimrod object title based on other meta-data in the Nimrod field and known use cases. """ - if field.field_code == 161 and field.threshold_value >= 0.: - field.title = "minimum_cloud_base_above_threshold" if field.field_code == 12: - field.title = "pressure" + field.title = "air_pressure" + if field.field_code == 27: + field.title = "snow fraction" if field.field_code == 28: field.title = "snow probability" if field.field_code == 29 and field.threshold_value >= 0.: @@ -116,32 +119,51 @@ def name(cube, field): field.title = "precipitation" if field.field_code == 63: field.title = "precipitation" - if field.field_code == 817: - field.title = "wind_speed_of_gust" + if field.field_code == 101: + field.title = "snow_melting_level_above_sea_level" + if field.field_code == 102: + field.title = "rain_melted_level_above_sea_level" if field.field_code == 155: field.title = "Visibility" + if field.field_code == 156: + field.title = "Worst visibility in grid point" + if field.field_code == 161 and field.threshold_value >= 0.: + field.title = "minimum_cloud_base_above_threshold" if field.field_code == 218: field.title = "snowfall" - if field.field_code == 101: - field.title = "snowmelt_above_sea_level" if field.field_code == 172: field.title = "cloud_area_fraction_in_atmosphere" if field.field_code == 421: field.title = "precipitation type" + if field.field_code == 501: + field.title = 'vector_wind_shear' + field.source = '' + if field.field_code == 508: + field.title = 'low_level_jet_u_component' + if field.field_code == 509: + field.title = 'low_level_jet_curvature' + if field.field_code == 514: + field.title = 'low_level_jet_v_component' if field.field_code == 804 and field.vertical_coord >= 0.: field.title = "wind speed" if field.field_code == 806 and field.vertical_coord >= 0.: field.title = "wind direction" + if field.field_code == 817: + field.title = "wind_speed_of_gust" if field.field_code == 821: field.title = "Probabilistic Gust Risk Analysis from Observations" field.source = "Nimrod pwind routine" if field.source.strip() == "pwind": field.source = "Nimrod pwind routine" - if getattr(field, "ensemble_member") == -98 and 'mean' not in field.title: - field.title = 'mean_of_' + field.title - if getattr(field, "ensemble_member") == -99 and 'spread' not in field.title: - field.title = 'standard_deviation_of_' + field.title + if getattr(field, "ensemble_member") == -98: + if 'mean' not in field.title: + field.title = 'mean_of_' + field.title + field.ensemble_member = field.int_mdi + if getattr(field, "ensemble_member") == -99: + if 'spread' not in field.title: + field.title = 'standard_deviation_of_' + field.title + field.ensemble_member = field.int_mdi cube.rename(field.title.strip()) @@ -167,8 +189,7 @@ def units(cube, field): 'logical': '', 'Code': '', 'mask': '', - 'oktas': '', - 'm/2-25k': '', + 'mb': 'hPa', 'g/Kg': '', 'unitless': '', 'Fraction': '1', @@ -178,6 +199,10 @@ def units(cube, field): 'n/a': ''} field_units = field.units.strip() + if field_units == 'm/2-25k': + # Handle strange visibility units + cube.data = (cube.data + 25000.) * 2 + field_units = 'm' if '*' in field_units: # Split into unit string and integer unit_list = field_units.split('*') @@ -200,6 +225,20 @@ def units(cube, field): if len(''.join(unit_list)) == 0: field_units = '1' cube.data = cube.data / 100. + if field_units == 'oktas': + field_units = '1' + cube.data /= 8. + if field_units == 'dBZ': + # cf_units doesn't recognise decibels (dBZ), but does know BZ + field_units = 'BZ' + cube.data /= 10. + if not field_units: + if field.field_code == 8: + # Relative Humidity data are unitless, but not "unknown" + field_units = '1' + if field.field_code in [505, 515]: + # CAPE units are not always set correctly. Assume J/kg + field_units = 'J/kg' if field_units in unit_exception_dictionary.keys(): field_units = unit_exception_dictionary[field_units] if len(field_units) > 0 and field_units[0] == '/': @@ -238,13 +277,13 @@ def time(cube, field): field.vt_minute, field.vt_second, ) - point = np.array(TIME_UNIT.date2num(valid_date), dtype=np.int64) + point = np.around(TIME_UNIT.date2num(valid_date)).astype(np.int64) lb_delta = None if field.period_minutes == 32767: - lb_delta = int(field.period_seconds) + lb_delta = field.period_seconds elif field.period_minutes != field.int_mdi and field.period_minutes != 0: - lb_delta = int(field.period_minutes) * 60 + lb_delta = field.period_minutes * 60 if lb_delta: bounds = np.array([point - lb_delta, point], dtype=np.int64) else: @@ -277,6 +316,38 @@ def reference_time(cube, field): cube.add_aux_coord(ref_time_coord) +def forecast_period(cube): + """ + Add a forecast_period coord based on existing time and + forecast_reference_time coords. + """ + try: + time_coord = cube.coord('time') + frt_coord = cube.coord('forecast_reference_time') + except CoordinateNotFoundError: + return + if len(time_coord.points) != 1 or len(frt_coord.points) != 1: + raise TranslationError( + "Unexpected number of points on time coordinates. Expected time:1; " + f"forecast_reference_time:1. Got {len(time_coord.points)}; " + f"{len(frt_coord.points)}") + time_delta = time_coord.cell(0).point - frt_coord.cell(0).point + + points = np.array(time_delta.days * 24 * 60 * 60 + time_delta.seconds, + dtype=np.int32) + forecast_period_unit = cf_units.Unit('second') + if cube.coord('time').has_bounds(): + time_window = time_coord.cell(0).bound + time_window = (time_window[1] - time_window[0]) + bounds = np.array([points - time_window.total_seconds(), points], + dtype=np.int32) + else: + bounds = None + cube.add_aux_coord( + iris.coords.AuxCoord(points, standard_name='forecast_period', + bounds=bounds, units=forecast_period_unit)) + + def mask_cube(cube, field): """ Updates cube.data to be a masked array if appropriate. @@ -433,16 +504,43 @@ def horizontal_grid(cube, field): def vertical_coord(cube, field): """Add a vertical coord to the cube, if appropriate.""" - coord_point = getattr(field, "vertical_coord") - if (coord_point == 9999. and - (field.reference_vertical_coord in [9999., field.int_mdi, - NIMROD_DEFAULT])): + if all([x in [field.int_mdi, NIMROD_DEFAULT] for x in [ + field.vertical_coord, field.vertical_coord_type, + field.reference_vertical_coord, + field.reference_vertical_coord_type]]): return - if coord_point in [9999., 8888.]: + + if (field.reference_vertical_coord_type not in [field.int_mdi, + NIMROD_DEFAULT] and + field.reference_vertical_coord_type != field.vertical_coord_type + and field.reference_vertical_coord not in [field.int_mdi, + NIMROD_DEFAULT]): + msg = ('Unmatched vertical coord types ' + f'{field.vertical_coord_type} != ' + f'{field.reference_vertical_coord_type}' + f'. Assuming {field.vertical_coord_type}') + warnings.warn(msg) + + coord_point = field.vertical_coord + if coord_point == 8888.: + if "sea_level" not in cube.name(): + cube.rename(f"{cube.name()}_at_mean_sea_level") + coord_point = 0. + if (field.reference_vertical_coord in [8888., field.int_mdi, + NIMROD_DEFAULT]): + # This describes a surface field. No changes needed. + return + + coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) + if coord_point == 9999.: + if (field.reference_vertical_coord in [9999., field.int_mdi, + NIMROD_DEFAULT]): + # This describes a surface field. No changes needed. + return # A bounded vertical coord starting from the surface - # Relies on correct field.vertical_coord_type to distinguish between - # these meanings. coord_point = 0. + coord_args = VERTICAL_CODES.get(field.reference_vertical_coord_type, + None) coord_point = np.array(coord_point, dtype=np.float32) if (field.reference_vertical_coord >= 0. and field.reference_vertical_coord != coord_point): @@ -451,7 +549,6 @@ def vertical_coord(cube, field): else: bounds = None - coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) if coord_args: new_coord = iris.coords.AuxCoord(coord_point, bounds=bounds, **coord_args) @@ -494,6 +591,8 @@ def add_attr(item): if hasattr(field, item): value = getattr(field, item) if value not in [field.int_mdi, field.float32_mdi]: + if 'radius' in item: + value = f'{value} km' cube.attributes[item] = value add_attr("nimrod_version") @@ -530,25 +629,38 @@ def add_attr(item): cube.attributes['source'] = source -def threshold_coord(cube, field): +def known_threshold_coord(field): """ - Adds a scalar threshold coord to the cube for known use cases. + Supplies known threshold coord meta-data for known use cases. """ - coord_keys = None + coord_keys = {} if field.field_code == 161 and field.threshold_value >= 0.: - coord_keys = {"standard_name": "cloud_area_fraction", - "units": ""} + coord_keys = {"var_name": "threshold"} + if field.threshold_value_alt > 8.: + coord_keys["standard_name"] = "height" + coord_keys["units"] = "metres" + else: + coord_keys["standard_name"] = "cloud_area_fraction" + coord_keys["units"] = "oktas" if field.field_code == 29 and field.threshold_value >= 0.: - coord_keys = {"standard_name": "visibility_in_air", - "units": "metres"} + if field.threshold_type == field.int_mdi: + coord_keys = {"standard_name": "visibility_in_air", + "var_name": "threshold", + "units": "metres"} + else: + coord_keys = {"long_name": "fog_fraction", + "var_name": "threshold", + "units": "1"} + if (field.field_code == 422 + and field.threshold_value >= 0. + and field.threshold_type == field.int_mdi): + coord_keys = {"long_name": "radius_of_max", + "units": "km"} if field.field_code == 821: coord_keys = {"standard_name": "wind_speed_of_gust", + "var_name": "threshold", "units": "m/s"} - if coord_keys: - cube.add_aux_coord(iris.coords.AuxCoord( - np.array(field.threshold_value, dtype=np.float32), - var_name='threshold', - **coord_keys)) + return coord_keys def probability_coord(cube, field): @@ -561,7 +673,7 @@ def probability_coord(cube, field): 'attributes': {'relative_to_threshold': 'above'}}, 2: {'var_name': 'threshold', 'attributes': {'relative_to_threshold': 'below'}}, - 3: {'standard_name': 'percentile', 'units': "%"}, + 3: {'long_name': 'percentile', 'units': "1"}, 4: {'var_name': 'threshold', 'attributes': {'relative_to_threshold': 'equal'}}} probmethod_lookup = {1: 'AOT (Any One Time)', @@ -570,9 +682,10 @@ def probability_coord(cube, field): 8: 'AOL (Any One Location)', 16: 'SW (Some Where)'} is_multi_member_field = False - coord_keys = probtype_lookup.get(field.threshold_type, None) - if not coord_keys: - return is_multi_member_field + coord_keys = probtype_lookup.get(field.threshold_type, {}) + if coord_keys: + is_multi_member_field = True + coord_keys.update(known_threshold_coord(field)) if not coord_keys.get('units', None): coord_keys['units'] = DEFAULT_UNITS.get(field.field_code, None) coord_val = None @@ -580,9 +693,9 @@ def probability_coord(cube, field): coord_val = field.threshold_value_alt elif field.threshold_value > -32766.: coord_val = field.threshold_value - if field.chead.find('pc') > 0: + if field.title.find('pc') > 0: try: - coord_val = [int(x.strip('pc')) for x in field.chead.split(' ') + coord_val = [int(x.strip('pc')) for x in field.title.split(' ') if x.find('pc') > 0][0] except IndexError: pass @@ -590,15 +703,25 @@ def probability_coord(cube, field): if field.threshold_fuzziness > -32766.: bounds = [coord_val * field.threshold_fuzziness, coord_val * (2. - field.threshold_fuzziness)] + bounds = np.array(bounds, dtype=np.float32) else: bounds = None + if coord_keys.get('units', None) == 'oktas': + coord_keys['units'] = '1' + coord_val /= 8. + if bounds is not None: + bounds /= 8. new_coord = iris.coords.AuxCoord( np.array(coord_val, dtype=np.float32), - bounds=np.array(bounds, dtype=np.float32), + bounds=bounds, **coord_keys) cube.add_aux_coord(new_coord) - cube.units = '1' - is_multi_member_field = True + if field.threshold_type == 3: + pass + else: + if is_multi_member_field: + cube.units = '1' + cube.rename(f'probability_of_{cube.name()}') if field.probability_method > 0: probability_attributes = [] @@ -658,6 +781,7 @@ def run(field): # time time(cube, field) reference_time(cube, field) + forecast_period(cube) experiment(cube, field) @@ -669,7 +793,6 @@ def run(field): # add other stuff, if present soil_type_coord(cube, field) - threshold_coord(cube, field) if not probability_coord(cube, field): ensemble_member(cube, field) time_averaging(cube, field) diff --git a/lib/iris/tests/results/nimrod/levels_below_ground.cml b/lib/iris/tests/results/nimrod/levels_below_ground.cml index 22d2168578..c7a5bb1713 100644 --- a/lib/iris/tests/results/nimrod/levels_below_ground.cml +++ b/lib/iris/tests/results/nimrod/levels_below_ground.cml @@ -3,11 +3,11 @@ - + - + diff --git a/lib/iris/tests/results/nimrod/load_2flds.cml b/lib/iris/tests/results/nimrod/load_2flds.cml index 9eeb754d66..196bce162f 100644 --- a/lib/iris/tests/results/nimrod/load_2flds.cml +++ b/lib/iris/tests/results/nimrod/load_2flds.cml @@ -1,6 +1,6 @@ - + @@ -8,6 +8,9 @@ + + + @@ -28,6 +31,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/mockography.cml b/lib/iris/tests/results/nimrod/mockography.cml index 6e63d9fa87..fd038c992e 100644 --- a/lib/iris/tests/results/nimrod/mockography.cml +++ b/lib/iris/tests/results/nimrod/mockography.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml new file mode 100644 index 0000000000..3a5c8bdc50 --- /dev/null +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -0,0 +1,1792 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml new file mode 100644 index 0000000000..66c118a908 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml new file mode 100644 index 0000000000..958253a812 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml new file mode 100644 index 0000000000..c7b0270bd8 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml new file mode 100644 index 0000000000..0298e63969 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml @@ -0,0 +1,279 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml new file mode 100644 index 0000000000..340baa5158 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml @@ -0,0 +1,361 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml new file mode 100644 index 0000000000..f7902ee33d --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml @@ -0,0 +1,267 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml new file mode 100644 index 0000000000..121bf9d4dd --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml @@ -0,0 +1,303 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml new file mode 100644 index 0000000000..79db42c407 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml new file mode 100644 index 0000000000..1dbbb05581 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml new file mode 100644 index 0000000000..8d099dbb5b --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml new file mode 100644 index 0000000000..1bc9b82719 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml @@ -0,0 +1,342 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml new file mode 100644 index 0000000000..67ddae1f24 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml new file mode 100644 index 0000000000..5fddeed7f4 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml @@ -0,0 +1,262 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml new file mode 100644 index 0000000000..c573185c51 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml new file mode 100644 index 0000000000..a95119a1f1 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml new file mode 100644 index 0000000000..e48d884f08 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml new file mode 100644 index 0000000000..2df3299af2 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml new file mode 100644 index 0000000000..b0ced2a7eb --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml new file mode 100644 index 0000000000..bb385a9912 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml new file mode 100644 index 0000000000..060eaa555f --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml new file mode 100644 index 0000000000..ebfd809071 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml @@ -0,0 +1,181 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml new file mode 100644 index 0000000000..52343a6857 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml @@ -0,0 +1,223 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml new file mode 100644 index 0000000000..7119f27212 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml new file mode 100644 index 0000000000..bfec1580fc --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml new file mode 100644 index 0000000000..8ce710d577 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml new file mode 100644 index 0000000000..93b02315cf --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml @@ -0,0 +1,228 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml new file mode 100644 index 0000000000..5298c02fe8 --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml new file mode 100644 index 0000000000..c417cc9eea --- /dev/null +++ b/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index c849543d2b..7922899310 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -37,6 +37,50 @@ def test_multi_field_load(self): ) self.assertCML(cube, ("nimrod", "load_2flds.cml")) + def test_huge_field_load(self): + # load a wide range of cubes with all meta-data variations + for datafile in {"u1096_ng_ek07_precip0540_accum180_18km", + "u1096_ng_ek00_cloud3d0060_2km", + "u1096_ng_ek00_cloud_2km", + "u1096_ng_ek00_convection_2km", + "u1096_ng_ek00_convwind_2km", + "u1096_ng_ek00_frzlev_2km", + "u1096_ng_ek00_height_2km", + "u1096_ng_ek00_precip_2km", + "u1096_ng_ek00_precipaccum_2km", + "u1096_ng_ek00_preciptype_2km", + "u1096_ng_ek00_pressure_2km", + "u1096_ng_ek00_radiation_2km", + "u1096_ng_ek00_radiationuv_2km", + "u1096_ng_ek00_refl_2km", + "u1096_ng_ek00_relhumidity3d0060_2km", + "u1096_ng_ek00_relhumidity_2km", + "u1096_ng_ek00_snow_2km", + "u1096_ng_ek00_soil3d0060_2km", + "u1096_ng_ek00_soil_2km", + "u1096_ng_ek00_temperature_2km", + "u1096_ng_ek00_visibility_2km", + "u1096_ng_ek00_wind_2km", + "u1096_ng_ek00_winduv3d0015_2km", + "u1096_ng_ek00_winduv_2km", + "u1096_ng_ek01_cape_2km", + "u1096_ng_umqv_fog_2km", + "u1096_ng_bmr04_precip_2km", + "u1096_ng_bsr05_precip_accum60_2km", + "probability_fields", + }: + cube = iris.load( + tests.get_data_path( + ( + "NIMROD", + "uk2km", + "cutouts", + datafile, + ) + ) + ) + self.assertCML(cube, ("nimrod", f"{datafile}.cml")) + def test_orography(self): # Mock an orography field we've seen. field = mock_nimrod_field() @@ -47,7 +91,11 @@ def test_orography(self): field.proj_biaxial_ellipsoid = 0 field.tm_meridian_scaling = 0.999601 field.field_code = 73 + field.reference_vertical_coord_type = field.int_mdi # Not bounded + field.reference_vertical_coord = field.int_mdi field.vertical_coord_type = 1 + field.vertical_coord = 8888 + field.ensemble_member = field.int_mdi field.title = "(MOCK) 2km mean orography" field.units = "metres" field.source = "GLOBE DTM" @@ -67,6 +115,7 @@ def test_levels_below_ground(self): field.field_code = -1 # Not orography field.reference_vertical_coord_type = field.int_mdi # Not bounded + field.reference_vertical_coord = field.int_mdi field.vertical_coord_type = 12 field.vertical_coord = 42 nimrod_load_rules.vertical_coord(cube, field) diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index dc666c50f8..4ccbc883c8 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -28,23 +28,34 @@ class Test(tests.IrisTest): def setUp(self): self.field = mock.Mock( + vertical_coord=NIMROD_DEFAULT, vertical_coord_type=NIMROD_DEFAULT, + reference_vertical_coord=NIMROD_DEFAULT, + reference_vertical_coord_type=NIMROD_DEFAULT, int_mdi=mock.sentinel.int_mdi, - field_code=mock.sentinel.field_code, - vertical_coord=mock.sentinel.vertical_coord, - reference_vertical_coord=mock.sentinel.reference_vertical_coord, - ensemble_member=NIMROD_DEFAULT, spec=NimrodField, ) self.cube = mock.Mock() - def _call_vertical_coord(self, vertical_coord_type): - self.field.vertical_coord_type = vertical_coord_type + def _call_vertical_coord(self, vertical_coord_val=None, + vertical_coord_type=None, + reference_vertical_coord=None, + reference_vertical_coord_type=None): + if vertical_coord_val: + self.field.vertical_coord = vertical_coord_val + if vertical_coord_type: + self.field.vertical_coord_type = vertical_coord_type + if reference_vertical_coord: + self.field.reference_vertical_coord = reference_vertical_coord + if reference_vertical_coord_type: + self.field.reference_vertical_coord_type = \ + reference_vertical_coord_type vertical_coord(self.cube, self.field) def test_unhandled(self): with mock.patch("warnings.warn") as warn: - self._call_vertical_coord(-1) + self._call_vertical_coord(vertical_coord_val=1., + vertical_coord_type=-1) warn.assert_called_once_with( "Vertical coord -1 not yet handled", TranslationWarning ) @@ -52,13 +63,20 @@ def test_unhandled(self): def test_height(self): name = "vertical_coord" with mock.patch(self.NIMROD_LOCATION + "." + name) as height: - self._call_vertical_coord(0) + self._call_vertical_coord(vertical_coord_val=1., + vertical_coord_type=0) height.assert_called_once_with(self.cube, self.field) def test_null(self): with mock.patch("warnings.warn") as warn: - self._call_vertical_coord(NIMROD_DEFAULT) - self._call_vertical_coord(self.field.int_mdi) + self._call_vertical_coord(vertical_coord_type=NIMROD_DEFAULT) + self._call_vertical_coord(vertical_coord_type=self.field.int_mdi) + self.assertEqual(warn.call_count, 0) + + def test_ground_level(self): + with mock.patch("warnings.warn") as warn: + self._call_vertical_coord(vertical_coord_val=9999., + vertical_coord_type=0) self.assertEqual(warn.call_count, 0) From f21d26de40fd65b6e0d03e3f03d297357bbf64ee Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 30 Jan 2020 15:37:46 +0000 Subject: [PATCH 07/46] Stickler issues solved. --- lib/iris/fileformats/nimrod_load_rules.py | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 2e704e3354..41d9b43ba2 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -383,25 +383,25 @@ def proj_biaxial_ellipsoid(field): Based firstly on the value given by ellipsoid, then by grid if ellipsoid is missing, select the right pre-defined ellipsoid dictionary (Airy_1830 or - International_1924). + international_1924). """ - # Reference for Airy_1830 and international_1924 ellipsoids: + # Reference for airy_1830 and international_1924 ellipsoids: # http://fcm9/projects/PostProc/wiki/PostProcDocDomains#ProjectionConstants # Reference for GRS80: - Airy_1830 = {'semi_major_axis': 6377563.396, 'semi_minor_axis': 6356256.910} - International_1924 = {'semi_major_axis': 6378388.000, + airy_1830 = {'semi_major_axis': 6377563.396, 'semi_minor_axis': 6356256.910} + international_1924 = {'semi_major_axis': 6378388.000, 'semi_minor_axis': 6356911.946} if field.proj_biaxial_ellipsoid == 0: - ellipsoid = Airy_1830 + ellipsoid = airy_1830 elif field.proj_biaxial_ellipsoid == 1: - ellipsoid = International_1924 + ellipsoid = international_1924 elif field.proj_biaxial_ellipsoid == field.int_mdi or \ field.proj_biaxial_ellipsoid == -32767: if field.horizontal_grid_type == 0: - ellipsoid = Airy_1830 + ellipsoid = airy_1830 elif field.horizontal_grid_type == 1 or field.horizontal_grid_type == 4: - ellipsoid = International_1924 + ellipsoid = international_1924 else: raise TranslationError('''Unsupported grid type, only NG, EuroPP and lat/long are possible''') @@ -448,8 +448,8 @@ def coord_system(field): set_british_national_grid_defaults(field) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: coord_sys = iris.coord_systems.TransverseMercator( - field.true_origin_latitude, field.true_origin_longitude, field.true_origin_easting, - field.true_origin_northing, + field.true_origin_latitude, field.true_origin_longitude, + field.true_origin_easting, field.true_origin_northing, field.tm_meridian_scaling, iris.coord_systems.GeogCS(**ellipsoid)) elif field.horizontal_grid_type == 1: coord_sys = iris.coord_systems.GeogCS(**ellipsoid) @@ -527,14 +527,14 @@ def vertical_coord(cube, field): cube.rename(f"{cube.name()}_at_mean_sea_level") coord_point = 0. if (field.reference_vertical_coord in [8888., field.int_mdi, - NIMROD_DEFAULT]): + NIMROD_DEFAULT]): # This describes a surface field. No changes needed. return coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) if coord_point == 9999.: if (field.reference_vertical_coord in [9999., field.int_mdi, - NIMROD_DEFAULT]): + NIMROD_DEFAULT]): # This describes a surface field. No changes needed. return # A bounded vertical coord starting from the surface From 7dd34b2f6aa8d15e3afe14a409980dcfc34c0735 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 30 Jan 2020 15:41:32 +0000 Subject: [PATCH 08/46] Updates iris-test-data commit reference. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9f6bf3bfe..6b51a18d1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ git: install: - > - export IRIS_TEST_DATA_REF="1696ac3a823a06b95f430670f285ee97671d2cf2"; + export IRIS_TEST_DATA_REF="06e3c7d289d4ae177539e71134a2a6f6a1bc8117"; export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//"); # Install miniconda From 96125f4e11895f39c76a99a73bbe42b39ab9377f Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 31 Jan 2020 09:23:14 +0000 Subject: [PATCH 09/46] Adds whats-new entry --- .../newfeature_2020-Jan-31_nimrod_format_enhancement.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Jan-31_nimrod_format_enhancement.txt diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Jan-31_nimrod_format_enhancement.txt b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Jan-31_nimrod_format_enhancement.txt new file mode 100644 index 0000000000..454fc3617f --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Jan-31_nimrod_format_enhancement.txt @@ -0,0 +1,3 @@ +* The :class:`~iris.fileformats.nimrod` provides richer meta-data translation +when loading Nimrod-format data into cubes. This covers most known operational +use-cases. From 62e56b19c9f035464ee8103cd00a7cae480c51dd Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 17 Feb 2020 14:32:01 +0000 Subject: [PATCH 10/46] Improves handling of Nimrod-format files as seen from Radarnet. - Improves handling of missing data values. - Improves handling for strings with null characters present. - Replaces instances of "unknown" units with Unit("1"). --- lib/iris/fileformats/nimrod_load_rules.py | 87 +++++++++++-------- .../nimrod/u1096_ng_ek00_preciptype_2km.cml | 6 +- .../nimrod_load_rules/test_vertical_coord.py | 1 + 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 41d9b43ba2..7b6f20deb9 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -7,6 +7,7 @@ import warnings import re +import string import cf_units import cftime @@ -100,6 +101,13 @@ class TranslationWarning(Warning): pass +def is_missing(field, value): + """Returns True if value matches an "is-missing" number.""" + return any(np.isclose(value, [field.int_mdi, + field.float32_mdi, + NIMROD_DEFAULT])) + + def name(cube, field): """Set the cube's name from the field. Modifies the Nimrod object title based on other meta-data in the @@ -165,8 +173,15 @@ def name(cube, field): field.title = 'standard_deviation_of_' + field.title field.ensemble_member = field.int_mdi - cube.rename(field.title.strip()) + cube.rename(remove_unprintable_chars(field.title)) + +def remove_unprintable_chars(input_str): + """ + Removes unprintable characters from a string and returns the result. + """ + return ''.join(c if c in string.printable else ' ' for c in + input_str).strip() def units(cube, field): """ @@ -186,19 +201,19 @@ def units(cube, field): unit_exception_dictionary = {'Knts': 'knots', 'knts': 'knots', 'J/Kg': 'J/kg', - 'logical': '', - 'Code': '', - 'mask': '', + 'logical': '1', + 'Code': '1', + 'mask': '1', 'mb': 'hPa', - 'g/Kg': '', - 'unitless': '', + 'g/Kg': '1', + 'unitless': '1', 'Fraction': '1', - 'index': '', + 'index': '1', 'Beaufort': '', 'mmh2o': 'kg/m2', - 'n/a': ''} + 'n/a': '1'} - field_units = field.units.strip() + field_units = remove_unprintable_chars(field.units) if field_units == 'm/2-25k': # Handle strange visibility units cube.data = (cube.data + 25000.) * 2 @@ -282,7 +297,8 @@ def time(cube, field): lb_delta = None if field.period_minutes == 32767: lb_delta = field.period_seconds - elif field.period_minutes != field.int_mdi and field.period_minutes != 0: + elif not is_missing(field, field.period_minutes) and \ + field.period_minutes != 0: lb_delta = field.period_minutes * 60 if lb_delta: bounds = np.array([point - lb_delta, point], dtype=np.int64) @@ -298,7 +314,7 @@ def time(cube, field): def reference_time(cube, field): """Add a 'reference time' to the cube, if present in the field.""" - if field.dt_year != field.int_mdi and field.dt_year > 0: + if not is_missing(field, field.dt_year) and field.dt_year > 0: data_date = cftime.datetime( field.dt_year, field.dt_month, @@ -370,7 +386,7 @@ def mask_cube(cube, field): def experiment(cube, field): """Add an 'experiment number' to the cube, if present in the field.""" - if field.experiment_num != field.int_mdi: + if not is_missing(field, field.experiment_num): cube.add_aux_coord( DimCoord(field.experiment_num, long_name="experiment_number") ) @@ -396,8 +412,7 @@ def proj_biaxial_ellipsoid(field): ellipsoid = airy_1830 elif field.proj_biaxial_ellipsoid == 1: ellipsoid = international_1924 - elif field.proj_biaxial_ellipsoid == field.int_mdi or \ - field.proj_biaxial_ellipsoid == -32767: + elif is_missing(field, field.proj_biaxial_ellipsoid): if field.horizontal_grid_type == 0: ellipsoid = airy_1830 elif field.horizontal_grid_type == 1 or field.horizontal_grid_type == 4: @@ -417,17 +432,16 @@ def set_british_national_grid_defaults(field): """Check for missing coord-system meta-data and set default values for the Ordnance Survey GB Transverse Mercator projection. Some Radarnet files are missing these.""" - invalid_values = (-32767., float(field.int_mdi)) - if np.any([field.true_origin_latitude == v for v in invalid_values]): + if is_missing(field, field.true_origin_latitude): field.true_origin_latitude = 49. - if np.any([field.true_origin_longitude == v for v in invalid_values]): + if is_missing(field, field.true_origin_longitude): field.true_origin_longitude = -2. - if np.any([field.true_origin_easting == v for v in invalid_values]): + if is_missing(field, field.true_origin_easting): field.true_origin_easting = 400000. - if np.any([field.true_origin_northing == v for v in invalid_values]): + if is_missing(field, field.true_origin_northing): field.true_origin_northing = -100000. - if np.any([field.tm_meridian_scaling == v for v in invalid_values]): + if is_missing(field, field.tm_meridian_scaling): field.tm_meridian_scaling = 0.9996012717 ng_central_meridian_sf_dp = 0.9996012717 @@ -504,17 +518,15 @@ def horizontal_grid(cube, field): def vertical_coord(cube, field): """Add a vertical coord to the cube, if appropriate.""" - if all([x in [field.int_mdi, NIMROD_DEFAULT] for x in [ + if all([is_missing(field, x) for x in [ field.vertical_coord, field.vertical_coord_type, field.reference_vertical_coord, field.reference_vertical_coord_type]]): return - if (field.reference_vertical_coord_type not in [field.int_mdi, - NIMROD_DEFAULT] and + if (not is_missing(field, field.reference_vertical_coord_type) and field.reference_vertical_coord_type != field.vertical_coord_type - and field.reference_vertical_coord not in [field.int_mdi, - NIMROD_DEFAULT]): + and not is_missing(field, field.reference_vertical_coord)): msg = ('Unmatched vertical coord types ' f'{field.vertical_coord_type} != ' f'{field.reference_vertical_coord_type}' @@ -526,15 +538,15 @@ def vertical_coord(cube, field): if "sea_level" not in cube.name(): cube.rename(f"{cube.name()}_at_mean_sea_level") coord_point = 0. - if (field.reference_vertical_coord in [8888., field.int_mdi, - NIMROD_DEFAULT]): + if (np.isclose(field.reference_vertical_coord, 8888.) or + is_missing(field, field.reference_vertical_coord)): # This describes a surface field. No changes needed. return coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) - if coord_point == 9999.: - if (field.reference_vertical_coord in [9999., field.int_mdi, - NIMROD_DEFAULT]): + if np.isclose(coord_point, 9999.): + if (np.isclose(field.reference_vertical_coord, 9999.) or + is_missing(field, field.reference_vertical_coord)): # This describes a surface field. No changes needed. return # A bounded vertical coord starting from the surface @@ -566,7 +578,7 @@ def vertical_coord(cube, field): def ensemble_member(cube, field): """Add an 'ensemble member' coord to the cube, if present in the field.""" ensemble_member_value = getattr(field, "ensemble_member") - if ensemble_member_value != field.int_mdi: + if not is_missing(field, ensemble_member_value): cube.add_aux_coord(DimCoord(np.array(ensemble_member_value, dtype=np.int32), "realization")) @@ -590,10 +602,11 @@ def add_attr(item): """Add an attribute to the cube.""" if hasattr(field, item): value = getattr(field, item) - if value not in [field.int_mdi, field.float32_mdi]: - if 'radius' in item: - value = f'{value} km' - cube.attributes[item] = value + if is_missing(field, value): + return + if 'radius' in item: + value = f'{value} km' + cube.attributes[item] = value add_attr("nimrod_version") add_attr("field_code") @@ -643,7 +656,7 @@ def known_threshold_coord(field): coord_keys["standard_name"] = "cloud_area_fraction" coord_keys["units"] = "oktas" if field.field_code == 29 and field.threshold_value >= 0.: - if field.threshold_type == field.int_mdi: + if is_missing(field, field.threshold_type): coord_keys = {"standard_name": "visibility_in_air", "var_name": "threshold", "units": "metres"} @@ -653,7 +666,7 @@ def known_threshold_coord(field): "units": "1"} if (field.field_code == 422 and field.threshold_value >= 0. - and field.threshold_type == field.int_mdi): + and is_missing(field, field.threshold_type)): coord_keys = {"long_name": "radius_of_max", "units": "km"} if field.field_code == 821: diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml index 1bc9b82719..61bbbb9240 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml @@ -226,7 +226,7 @@ - + @@ -263,7 +263,7 @@ - + @@ -301,7 +301,7 @@ - + diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index 4ccbc883c8..58e012f0e0 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -33,6 +33,7 @@ def setUp(self): reference_vertical_coord=NIMROD_DEFAULT, reference_vertical_coord_type=NIMROD_DEFAULT, int_mdi=mock.sentinel.int_mdi, + float32_mdi=NIMROD_DEFAULT, spec=NimrodField, ) self.cube = mock.Mock() From 8a9c9059e013d3d8139a4cbff9df35183e086217 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 17 Feb 2020 14:34:35 +0000 Subject: [PATCH 11/46] Corrects indentation for Stickler Bot. --- lib/iris/fileformats/nimrod_load_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 7b6f20deb9..c33ebc5ee4 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -546,7 +546,7 @@ def vertical_coord(cube, field): coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) if np.isclose(coord_point, 9999.): if (np.isclose(field.reference_vertical_coord, 9999.) or - is_missing(field, field.reference_vertical_coord)): + is_missing(field, field.reference_vertical_coord)): # This describes a surface field. No changes needed. return # A bounded vertical coord starting from the surface From 1e9b92267adcec0646513d4b2764def3e7a5d0ec Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 10:09:13 +0000 Subject: [PATCH 12/46] Updates handling of specific attributes. - Adds title and institution attributes. - Excludes some attributes when handling radar data as these have different meanings. --- lib/iris/fileformats/nimrod_load_rules.py | 11 ++- lib/iris/tests/results/nimrod/load_2flds.cml | 2 + lib/iris/tests/results/nimrod/mockography.cml | 2 + .../results/nimrod/probability_fields.cml | 84 +++++++++++++++++++ .../nimrod/u1096_ng_bmr04_precip_2km.cml | 2 + .../u1096_ng_bsr05_precip_accum60_2km.cml | 2 + .../nimrod/u1096_ng_ek00_cloud3d0060_2km.cml | 4 + .../nimrod/u1096_ng_ek00_cloud_2km.cml | 14 ++++ .../nimrod/u1096_ng_ek00_convection_2km.cml | 18 ++++ .../nimrod/u1096_ng_ek00_convwind_2km.cml | 12 +++ .../nimrod/u1096_ng_ek00_frzlev_2km.cml | 16 ++++ .../nimrod/u1096_ng_ek00_height_2km.cml | 2 + .../nimrod/u1096_ng_ek00_precip_2km.cml | 6 ++ .../nimrod/u1096_ng_ek00_precipaccum_2km.cml | 2 + .../nimrod/u1096_ng_ek00_preciptype_2km.cml | 18 ++++ .../nimrod/u1096_ng_ek00_pressure_2km.cml | 4 + .../nimrod/u1096_ng_ek00_radiation_2km.cml | 14 ++++ .../nimrod/u1096_ng_ek00_radiationuv_2km.cml | 6 ++ .../results/nimrod/u1096_ng_ek00_refl_2km.cml | 2 + .../u1096_ng_ek00_relhumidity3d0060_2km.cml | 2 + .../nimrod/u1096_ng_ek00_relhumidity_2km.cml | 2 + .../results/nimrod/u1096_ng_ek00_snow_2km.cml | 6 ++ .../nimrod/u1096_ng_ek00_soil3d0060_2km.cml | 8 ++ .../results/nimrod/u1096_ng_ek00_soil_2km.cml | 6 ++ .../nimrod/u1096_ng_ek00_temperature_2km.cml | 8 ++ .../nimrod/u1096_ng_ek00_visibility_2km.cml | 10 +++ .../results/nimrod/u1096_ng_ek00_wind_2km.cml | 12 +++ .../nimrod/u1096_ng_ek00_winduv3d0015_2km.cml | 4 + .../nimrod/u1096_ng_ek00_winduv_2km.cml | 4 + .../results/nimrod/u1096_ng_ek01_cape_2km.cml | 12 +++ ...u1096_ng_ek07_precip0540_accum180_18km.cml | 2 + .../results/nimrod/u1096_ng_umqv_fog_2km.cml | 2 + .../nimrod_load_rules/test_vertical_coord.py | 2 +- 33 files changed, 296 insertions(+), 5 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index c33ebc5ee4..499d402748 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -629,10 +629,11 @@ def add_attr(item): add_attr("sensor_id") add_attr("meteosat_id") add_attr("alphas_available") - for key in ["neighbourhood_radius", "recursive_filter_iterations", - "recursive_filter_alpha", "threshold_vicinity_radius", - "probability_period_of_event"]: - add_attr(key) + if 'radar' not in field.source: + for key in ["neighbourhood_radius", "recursive_filter_iterations", + "recursive_filter_alpha", "threshold_vicinity_radius", + "probability_period_of_event"]: + add_attr(key) source = field.source.strip() rematcher = re.compile('^ek\d\d$') @@ -640,6 +641,8 @@ def add_attr(item): or source.find('umek') == 0): source = 'MOGREPS-UK' cube.attributes['source'] = source + cube.attributes['title'] = 'Unknown' + cube.attributes['institution'] = 'Met Office' def known_threshold_coord(field): diff --git a/lib/iris/tests/results/nimrod/load_2flds.cml b/lib/iris/tests/results/nimrod/load_2flds.cml index 196bce162f..72f06f8cd1 100644 --- a/lib/iris/tests/results/nimrod/load_2flds.cml +++ b/lib/iris/tests/results/nimrod/load_2flds.cml @@ -3,9 +3,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/mockography.cml b/lib/iris/tests/results/nimrod/mockography.cml index fd038c992e..585ef2b952 100644 --- a/lib/iris/tests/results/nimrod/mockography.cml +++ b/lib/iris/tests/results/nimrod/mockography.cml @@ -3,7 +3,9 @@ + + diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml index 3a5c8bdc50..223c3e88a9 100644 --- a/lib/iris/tests/results/nimrod/probability_fields.cml +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -3,9 +3,11 @@ + + @@ -46,9 +48,11 @@ + + @@ -93,9 +97,11 @@ + + @@ -135,12 +141,14 @@ + + @@ -188,9 +196,11 @@ + + @@ -237,12 +247,14 @@ + + @@ -289,9 +301,11 @@ + + @@ -327,12 +341,14 @@ + + @@ -372,9 +388,11 @@ + + @@ -413,9 +431,11 @@ + + @@ -450,9 +470,11 @@ + + @@ -485,9 +507,11 @@ + + @@ -522,9 +546,11 @@ + + @@ -557,12 +583,14 @@ + + @@ -602,9 +630,11 @@ + + @@ -643,10 +673,12 @@ + + @@ -679,10 +711,12 @@ + + @@ -717,10 +751,12 @@ + + @@ -753,6 +789,7 @@ + @@ -760,6 +797,7 @@ + @@ -800,10 +838,12 @@ + + @@ -843,6 +883,7 @@ + @@ -850,6 +891,7 @@ + @@ -890,12 +932,14 @@ + + @@ -934,9 +978,11 @@ + + @@ -968,10 +1014,12 @@ + + @@ -1004,10 +1052,12 @@ + + @@ -1043,12 +1093,14 @@ + + @@ -1087,9 +1139,11 @@ + + @@ -1122,9 +1176,11 @@ + + @@ -1166,9 +1222,11 @@ + + @@ -1208,9 +1266,11 @@ + + @@ -1252,9 +1312,11 @@ + + @@ -1293,9 +1355,11 @@ + + @@ -1335,6 +1399,7 @@ + @@ -1342,6 +1407,7 @@ + @@ -1388,9 +1454,11 @@ + + @@ -1436,9 +1504,11 @@ + + @@ -1478,9 +1548,11 @@ + + @@ -1522,9 +1594,11 @@ + + @@ -1564,6 +1638,7 @@ + @@ -1571,6 +1646,7 @@ + @@ -1617,9 +1693,11 @@ + + @@ -1665,9 +1743,11 @@ + + @@ -1706,9 +1786,11 @@ + + @@ -1748,9 +1830,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml index 66c118a908..31518dd321 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_bmr04_precip_2km.cml @@ -3,10 +3,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml index 958253a812..80cb1834c0 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_bsr05_precip_accum60_2km.cml @@ -3,10 +3,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml index c7b0270bd8..5ef167f1ac 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml @@ -3,9 +3,11 @@ + + @@ -57,9 +59,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml index 0298e63969..5a43bc8596 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + @@ -77,10 +81,12 @@ + + @@ -115,10 +121,12 @@ + + @@ -153,9 +161,11 @@ + + @@ -202,9 +212,11 @@ + + @@ -239,9 +251,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml index 340baa5158..a98f3d4147 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml @@ -3,9 +3,11 @@ + + @@ -47,9 +49,11 @@ + + @@ -84,9 +88,11 @@ + + @@ -121,9 +127,11 @@ + + @@ -159,6 +167,7 @@ + @@ -166,6 +175,7 @@ + @@ -207,9 +217,11 @@ + + @@ -244,9 +256,11 @@ + + @@ -281,9 +295,11 @@ + + @@ -321,9 +337,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml index f7902ee33d..610dbd951f 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml @@ -3,9 +3,11 @@ + + @@ -47,9 +49,11 @@ + + @@ -91,9 +95,11 @@ + + @@ -135,9 +141,11 @@ + + @@ -179,9 +187,11 @@ + + @@ -223,9 +233,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml index 121bf9d4dd..f039e12d8b 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + @@ -77,9 +81,11 @@ + + @@ -114,10 +120,12 @@ + + @@ -152,10 +160,12 @@ + + @@ -190,9 +200,11 @@ + + @@ -227,10 +239,12 @@ + + @@ -265,10 +279,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml index 79db42c407..e2dc992c15 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml @@ -3,9 +3,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml index 1dbbb05581..d79dd83f55 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,10 +42,12 @@ + + @@ -78,10 +82,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml index 8d099dbb5b..8268297e22 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml @@ -3,10 +3,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml index 61bbbb9240..701116d0c7 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,10 +42,12 @@ + + @@ -78,10 +82,12 @@ + + @@ -116,9 +122,11 @@ + + @@ -153,10 +161,12 @@ + + @@ -191,10 +201,12 @@ + + @@ -229,9 +241,11 @@ + + @@ -266,10 +280,12 @@ + + @@ -304,10 +320,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml index 67ddae1f24..9d354ec469 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml index 5fddeed7f4..eebff3179d 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + @@ -77,9 +81,11 @@ + + @@ -114,9 +120,11 @@ + + @@ -151,9 +159,11 @@ + + @@ -188,9 +198,11 @@ + + @@ -225,9 +237,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml index c573185c51..de10bd934c 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + @@ -77,9 +81,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml index a95119a1f1..7d8bd30fdb 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml @@ -3,9 +3,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml index e48d884f08..bf2f96a5f6 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml @@ -3,9 +3,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml index 2df3299af2..9b7e7582d0 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity_2km.cml @@ -3,9 +3,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml index b0ced2a7eb..b2bfe10e9c 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,10 +42,12 @@ + + @@ -78,9 +82,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml index bb385a9912..0c257197d6 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml @@ -3,9 +3,11 @@ + + @@ -47,9 +49,11 @@ + + @@ -91,9 +95,11 @@ + + @@ -135,9 +141,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml index 060eaa555f..1bdda8f9cf 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + @@ -82,9 +86,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml index ebfd809071..354e4d7be0 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml @@ -3,10 +3,12 @@ + + @@ -48,10 +50,12 @@ + + @@ -93,9 +97,11 @@ + + @@ -137,9 +143,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml index 52343a6857..f2a9e7cfa4 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml @@ -3,9 +3,11 @@ + + @@ -47,9 +49,11 @@ + + @@ -91,9 +95,11 @@ + + @@ -135,9 +141,11 @@ + + @@ -179,9 +187,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml index 7119f27212..6e6b2c0b69 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml @@ -3,10 +3,12 @@ + + @@ -48,9 +50,11 @@ + + @@ -92,9 +96,11 @@ + + @@ -136,9 +142,11 @@ + + @@ -180,9 +188,11 @@ + + @@ -224,10 +234,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml index bfec1580fc..b3177568ae 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml @@ -3,9 +3,11 @@ + + @@ -47,9 +49,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml index 8ce710d577..aa14346e2f 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv_2km.cml @@ -3,9 +3,11 @@ + + @@ -47,9 +49,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml index 93b02315cf..bfb716a8fc 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml @@ -3,9 +3,11 @@ + + @@ -40,9 +42,11 @@ + + @@ -77,9 +81,11 @@ + + @@ -117,9 +123,11 @@ + + @@ -154,9 +162,11 @@ + + @@ -191,9 +201,11 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml index 5298c02fe8..7ad5bda993 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml @@ -3,10 +3,12 @@ + + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml index c417cc9eea..57756ccc1d 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_umqv_fog_2km.cml @@ -3,9 +3,11 @@ + + diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index 58e012f0e0..01a64d7566 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -32,7 +32,7 @@ def setUp(self): vertical_coord_type=NIMROD_DEFAULT, reference_vertical_coord=NIMROD_DEFAULT, reference_vertical_coord_type=NIMROD_DEFAULT, - int_mdi=mock.sentinel.int_mdi, + int_mdi=-32767, float32_mdi=NIMROD_DEFAULT, spec=NimrodField, ) From 1e4eb5839b04cd4bb020ada2bc6c8c3b81de18dd Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 10:28:31 +0000 Subject: [PATCH 13/46] Tries changing the source of the iris-test data. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6b51a18d1d..63c463dd57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,7 +84,7 @@ install: # iris test data - > if [[ "${TEST_MINIMAL}" != true ]]; then - wget --quiet -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; + wget --quiet -O iris-test-data.zip https://github.com/MoseleyS/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; unzip -q iris-test-data.zip; mv "iris-test-data-${IRIS_TEST_DATA_SUFFIX}" iris-test-data; fi From 28cf783e7b88116db5000e3b016355e888377376 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 10:49:35 +0000 Subject: [PATCH 14/46] Adds decorator to skip data tests when TEST_MINIMAL=True. --- .travis.yml | 2 +- lib/iris/tests/test_nimrod.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 63c463dd57..6b51a18d1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,7 +84,7 @@ install: # iris test data - > if [[ "${TEST_MINIMAL}" != true ]]; then - wget --quiet -O iris-test-data.zip https://github.com/MoseleyS/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; + wget --quiet -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; unzip -q iris-test-data.zip; mv "iris-test-data-${IRIS_TEST_DATA_SUFFIX}" iris-test-data; fi diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 7922899310..d165c46ca0 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -37,6 +37,7 @@ def test_multi_field_load(self): ) self.assertCML(cube, ("nimrod", "load_2flds.cml")) + @tests.skip_data def test_huge_field_load(self): # load a wide range of cubes with all meta-data variations for datafile in {"u1096_ng_ek07_precip0540_accum180_18km", From c97d04821b2b400f87a36a2db71e503035d6518f Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 21 Feb 2020 12:36:11 +0000 Subject: [PATCH 15/46] Removes unnecessary test. --- .../fileformats/nimrod_load_rules/test_vertical_coord.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index 01a64d7566..48f03049d1 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -61,13 +61,6 @@ def test_unhandled(self): "Vertical coord -1 not yet handled", TranslationWarning ) - def test_height(self): - name = "vertical_coord" - with mock.patch(self.NIMROD_LOCATION + "." + name) as height: - self._call_vertical_coord(vertical_coord_val=1., - vertical_coord_type=0) - height.assert_called_once_with(self.cube, self.field) - def test_null(self): with mock.patch("warnings.warn") as warn: self._call_vertical_coord(vertical_coord_type=NIMROD_DEFAULT) From ae1d3a15a92759b6771fe459d277de5edb33104e Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 21 Feb 2020 15:51:47 +0000 Subject: [PATCH 16/46] Corrects crs origin information in some old Nimrod files. --- lib/iris/fileformats/nimrod_load_rules.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 499d402748..2af58a89ae 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -437,9 +437,13 @@ def set_british_national_grid_defaults(field): field.true_origin_latitude = 49. if is_missing(field, field.true_origin_longitude): field.true_origin_longitude = -2. - if is_missing(field, field.true_origin_easting): + if is_missing(field, field.true_origin_easting) or np.isclose( + # Some old files misquote the value in km instead of m + field.true_origin_easting, 400.): field.true_origin_easting = 400000. - if is_missing(field, field.true_origin_northing): + if is_missing(field, field.true_origin_northing) or np.isclose( + # Some old files misquote the value in km instead of m + field.true_origin_northing, -100.): field.true_origin_northing = -100000. if is_missing(field, field.tm_meridian_scaling): field.tm_meridian_scaling = 0.9996012717 From f7c4ae992aa6303ce89d523ab7d28ea5e5005299 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 24 Feb 2020 14:03:25 +0000 Subject: [PATCH 17/46] Blacker than the ace of spades. Ran black on the code base and committed the results. I think I've blacked-out. --- lib/iris/fileformats/nimrod.py | 66 +- lib/iris/fileformats/nimrod_load_rules.py | 623 ++++++++++-------- lib/iris/tests/test_nimrod.py | 70 +- .../nimrod_load_rules/test_vertical_coord.py | 24 +- 4 files changed, 438 insertions(+), 345 deletions(-) diff --git a/lib/iris/fileformats/nimrod.py b/lib/iris/fileformats/nimrod.py index 2731c4fb27..7d203030c5 100644 --- a/lib/iris/fileformats/nimrod.py +++ b/lib/iris/fileformats/nimrod.py @@ -89,39 +89,39 @@ "sat_space_count", "ducting_index", "elevation_angle", - 'neighbourhood_radius', - 'threshold_vicinity_radius', - 'recursive_filter_alpha', - 'threshold_fuzziness', - 'threshold_duration_fuzziness', - 'data_header_float32_05', - 'data_header_float32_06', - 'data_header_float32_07', - 'data_header_float32_08', - 'data_header_float32_09', - 'data_header_float32_10', - 'data_header_float32_11', - 'data_header_float32_12', - 'data_header_float32_13', - 'data_header_float32_14', - 'data_header_float32_15', - 'data_header_float32_16', - 'data_header_float32_17', - 'data_header_float32_18', - 'data_header_float32_19', - 'data_header_float32_20', - 'data_header_float32_21', - 'data_header_float32_22', - 'data_header_float32_23', - 'data_header_float32_24', - 'data_header_float32_25', - 'data_header_float32_26', - 'data_header_float32_27', - 'data_header_float32_28', - 'data_header_float32_29', - 'data_header_float32_30', - 'data_header_float32_31', - 'data_header_float32_32', + "neighbourhood_radius", + "threshold_vicinity_radius", + "recursive_filter_alpha", + "threshold_fuzziness", + "threshold_duration_fuzziness", + "data_header_float32_05", + "data_header_float32_06", + "data_header_float32_07", + "data_header_float32_08", + "data_header_float32_09", + "data_header_float32_10", + "data_header_float32_11", + "data_header_float32_12", + "data_header_float32_13", + "data_header_float32_14", + "data_header_float32_15", + "data_header_float32_16", + "data_header_float32_17", + "data_header_float32_18", + "data_header_float32_19", + "data_header_float32_20", + "data_header_float32_21", + "data_header_float32_22", + "data_header_float32_23", + "data_header_float32_24", + "data_header_float32_25", + "data_header_float32_26", + "data_header_float32_27", + "data_header_float32_28", + "data_header_float32_29", + "data_header_float32_30", + "data_header_float32_31", + "data_header_float32_32", ) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 2af58a89ae..a6be41fb56 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -32,30 +32,46 @@ ) -DEFAULT_UNITS = {817: 'm s^-1', - 804: 'knots', - 422: 'min^-1', - 218: 'cm', - 172: 'oktas', - 155: 'm', - 101: 'm', - 63: 'mm hr^-1', - 61: 'mm', - 58: 'Celsius', - 12: 'mb', - 8: '1'} +DEFAULT_UNITS = { + 817: "m s^-1", + 804: "knots", + 422: "min^-1", + 218: "cm", + 172: "oktas", + 155: "m", + 101: "m", + 63: "mm hr^-1", + 61: "mm", + 58: "Celsius", + 12: "mb", + 8: "1", +} FIELD_CODES = {73: "orography"} # VERTICAL_CODES contains conversions from the Nimrod Documentation for the # header entry 20 for the vertical coordinate type -VERTICAL_CODES = {0: {'standard_name': 'height', 'units': 'm', - 'attributes': {"positive": "up"}}, - 1: {'standard_name': 'altitude', 'units': 'm', - 'attributes': {"positive": "up"}}, - 2: {'standard_name': 'air_pressure', 'units': 'hPa', - 'attributes': {"positive": "down"}}, - 6: {'standard_name': 'air_temperature', 'units': 'K'}, - 12: {'long_name': 'depth_below_ground', 'units': 'm', - 'attributes': {"positive": "down"}}} +VERTICAL_CODES = { + 0: { + "standard_name": "height", + "units": "m", + "attributes": {"positive": "up"}, + }, + 1: { + "standard_name": "altitude", + "units": "m", + "attributes": {"positive": "up"}, + }, + 2: { + "standard_name": "air_pressure", + "units": "hPa", + "attributes": {"positive": "down"}, + }, + 6: {"standard_name": "air_temperature", "units": "K"}, + 12: { + "long_name": "depth_below_ground", + "units": "m", + "attributes": {"positive": "down"}, + }, +} # Unhandled VERTICAL_CODES values (no use case identified): # 3: ['sigma', 'model level'], # 4: ['eta', 'model level'], @@ -66,34 +82,35 @@ # 10: ['potential vorticity', 'unknown'], # 11: ['cloud boundary', 'unknown'], -SOIL_TYPE_CODES = {1: "broadleaf_tree", - 2: "needleleaf_tree", - 3: "c3_grass", - 4: "c4_grass", - 5: "crop", - 6: "shrub", - 7: "urban", - 8: "water", - 9: "soil", - 10: "ice", - 601: "urban_canyon", - 602: "urban_roof", - } +SOIL_TYPE_CODES = { + 1: "broadleaf_tree", + 2: "needleleaf_tree", + 3: "c3_grass", + 4: "c4_grass", + 5: "crop", + 6: "shrub", + 7: "urban", + 8: "water", + 9: "soil", + 10: "ice", + 601: "urban_canyon", + 602: "urban_roof", +} TIME_AVERAGING_CODES = { 8192: "maximum in period", - 4096: 'minimum in period', - 2048: 'unknown(2048)', - 1024: 'unknown(1024)', - 512: 'time lagged', - 256: 'extrapolation', - 128: 'accumulation or average', - 64: 'from UM 150m', - 32: 'scaled to UM resolution', - 16: 'averaged over multiple surface types', - 8: 'only observations used', - 4: 'smoothed', - 2: 'cold bias applied', - 1: 'warm bias applied', + 4096: "minimum in period", + 2048: "unknown(2048)", + 1024: "unknown(1024)", + 512: "time lagged", + 256: "extrapolation", + 128: "accumulation or average", + 64: "from UM 150m", + 32: "scaled to UM resolution", + 16: "averaged over multiple surface types", + 8: "only observations used", + 4: "smoothed", + 2: "cold bias applied", + 1: "warm bias applied", } @@ -103,9 +120,9 @@ class TranslationWarning(Warning): def is_missing(field, value): """Returns True if value matches an "is-missing" number.""" - return any(np.isclose(value, [field.int_mdi, - field.float32_mdi, - NIMROD_DEFAULT])) + return any( + np.isclose(value, [field.int_mdi, field.float32_mdi, NIMROD_DEFAULT]) + ) def name(cube, field): @@ -119,7 +136,7 @@ def name(cube, field): field.title = "snow fraction" if field.field_code == 28: field.title = "snow probability" - if field.field_code == 29 and field.threshold_value >= 0.: + if field.field_code == 29 and field.threshold_value >= 0.0: field.title = "fog fraction" if field.field_code == 58: field.title = "temperature" @@ -135,7 +152,7 @@ def name(cube, field): field.title = "Visibility" if field.field_code == 156: field.title = "Worst visibility in grid point" - if field.field_code == 161 and field.threshold_value >= 0.: + if field.field_code == 161 and field.threshold_value >= 0.0: field.title = "minimum_cloud_base_above_threshold" if field.field_code == 218: field.title = "snowfall" @@ -144,17 +161,17 @@ def name(cube, field): if field.field_code == 421: field.title = "precipitation type" if field.field_code == 501: - field.title = 'vector_wind_shear' - field.source = '' + field.title = "vector_wind_shear" + field.source = "" if field.field_code == 508: - field.title = 'low_level_jet_u_component' + field.title = "low_level_jet_u_component" if field.field_code == 509: - field.title = 'low_level_jet_curvature' + field.title = "low_level_jet_curvature" if field.field_code == 514: - field.title = 'low_level_jet_v_component' - if field.field_code == 804 and field.vertical_coord >= 0.: + field.title = "low_level_jet_v_component" + if field.field_code == 804 and field.vertical_coord >= 0.0: field.title = "wind speed" - if field.field_code == 806 and field.vertical_coord >= 0.: + if field.field_code == 806 and field.vertical_coord >= 0.0: field.title = "wind direction" if field.field_code == 817: field.title = "wind_speed_of_gust" @@ -165,12 +182,12 @@ def name(cube, field): field.source = "Nimrod pwind routine" if getattr(field, "ensemble_member") == -98: - if 'mean' not in field.title: - field.title = 'mean_of_' + field.title + if "mean" not in field.title: + field.title = "mean_of_" + field.title field.ensemble_member = field.int_mdi if getattr(field, "ensemble_member") == -99: - if 'spread' not in field.title: - field.title = 'standard_deviation_of_' + field.title + if "spread" not in field.title: + field.title = "standard_deviation_of_" + field.title field.ensemble_member = field.int_mdi cube.rename(remove_unprintable_chars(field.title)) @@ -180,8 +197,10 @@ def remove_unprintable_chars(input_str): """ Removes unprintable characters from a string and returns the result. """ - return ''.join(c if c in string.printable else ' ' for c in - input_str).strip() + return "".join( + c if c in string.printable else " " for c in input_str + ).strip() + def units(cube, field): """ @@ -198,75 +217,79 @@ def units(cube, field): Unhandled units are stored in an "invalid_units" attribute instead. """ - unit_exception_dictionary = {'Knts': 'knots', - 'knts': 'knots', - 'J/Kg': 'J/kg', - 'logical': '1', - 'Code': '1', - 'mask': '1', - 'mb': 'hPa', - 'g/Kg': '1', - 'unitless': '1', - 'Fraction': '1', - 'index': '1', - 'Beaufort': '', - 'mmh2o': 'kg/m2', - 'n/a': '1'} + unit_exception_dictionary = { + "Knts": "knots", + "knts": "knots", + "J/Kg": "J/kg", + "logical": "1", + "Code": "1", + "mask": "1", + "mb": "hPa", + "g/Kg": "1", + "unitless": "1", + "Fraction": "1", + "index": "1", + "Beaufort": "", + "mmh2o": "kg/m2", + "n/a": "1", + } field_units = remove_unprintable_chars(field.units) - if field_units == 'm/2-25k': + if field_units == "m/2-25k": # Handle strange visibility units - cube.data = (cube.data + 25000.) * 2 - field_units = 'm' - if '*' in field_units: + cube.data = (cube.data + 25000.0) * 2 + field_units = "m" + if "*" in field_units: # Split into unit string and integer - unit_list = field_units.split('*') - if '^' in unit_list[1]: + unit_list = field_units.split("*") + if "^" in unit_list[1]: # Split out magnitude - unit_sublist = unit_list[1].split('^') + unit_sublist = unit_list[1].split("^") cube.data = cube.data / float(unit_sublist[0]) ** float( - unit_sublist[1]) + unit_sublist[1] + ) else: cube.data = cube.data / float(unit_list[1]) field_units = unit_list[0] - if 'ug/m3E1' in field_units: + if "ug/m3E1" in field_units: # Split into unit string and integer - unit_list = field_units.split('E') - cube.data = cube.data / 10.**float(unit_list[1]) + unit_list = field_units.split("E") + cube.data = cube.data / 10.0 ** float(unit_list[1]) field_units = unit_list[0] - if '%' in field_units: + if "%" in field_units: # Convert any percentages into fraction - unit_list = field_units.split('%') - if len(''.join(unit_list)) == 0: - field_units = '1' - cube.data = cube.data / 100. - if field_units == 'oktas': - field_units = '1' - cube.data /= 8. - if field_units == 'dBZ': + unit_list = field_units.split("%") + if len("".join(unit_list)) == 0: + field_units = "1" + cube.data = cube.data / 100.0 + if field_units == "oktas": + field_units = "1" + cube.data /= 8.0 + if field_units == "dBZ": # cf_units doesn't recognise decibels (dBZ), but does know BZ - field_units = 'BZ' - cube.data /= 10. + field_units = "BZ" + cube.data /= 10.0 if not field_units: if field.field_code == 8: # Relative Humidity data are unitless, but not "unknown" - field_units = '1' + field_units = "1" if field.field_code in [505, 515]: # CAPE units are not always set correctly. Assume J/kg - field_units = 'J/kg' + field_units = "J/kg" if field_units in unit_exception_dictionary.keys(): field_units = unit_exception_dictionary[field_units] - if len(field_units) > 0 and field_units[0] == '/': + if len(field_units) > 0 and field_units[0] == "/": # Deal with the case where the units are of the form '/unit' eg # '/second' in the Nimrod file. This converts to the form unit^-1 - field_units = field_units[1:] + '^-1' + field_units = field_units[1:] + "^-1" try: cube.units = field_units except ValueError: # Just add it as an attribute. warnings.warn( "Unhandled units '{0}' recorded in cube attributes.".format( - field_units) + field_units + ) ) cube.attributes["invalid_units"] = field_units @@ -281,8 +304,7 @@ def time(cube, field): # although if the valid time is missing too, it will be # made to be the same, so the forecast_period will always # be zero for these files. - valid_date = cftime.datetime( - 2016, 1, 1, 0, 0, 0) + valid_date = cftime.datetime(2016, 1, 1, 0, 0, 0) else: valid_date = cftime.datetime( field.vt_year, @@ -297,8 +319,10 @@ def time(cube, field): lb_delta = None if field.period_minutes == 32767: lb_delta = field.period_seconds - elif not is_missing(field, field.period_minutes) and \ - field.period_minutes != 0: + elif ( + not is_missing(field, field.period_minutes) + and field.period_minutes != 0 + ): lb_delta = field.period_minutes * 60 if lb_delta: bounds = np.array([point - lb_delta, point], dtype=np.int64) @@ -338,30 +362,38 @@ def forecast_period(cube): forecast_reference_time coords. """ try: - time_coord = cube.coord('time') - frt_coord = cube.coord('forecast_reference_time') + time_coord = cube.coord("time") + frt_coord = cube.coord("forecast_reference_time") except CoordinateNotFoundError: return if len(time_coord.points) != 1 or len(frt_coord.points) != 1: raise TranslationError( "Unexpected number of points on time coordinates. Expected time:1; " f"forecast_reference_time:1. Got {len(time_coord.points)}; " - f"{len(frt_coord.points)}") + f"{len(frt_coord.points)}" + ) time_delta = time_coord.cell(0).point - frt_coord.cell(0).point - points = np.array(time_delta.days * 24 * 60 * 60 + time_delta.seconds, - dtype=np.int32) - forecast_period_unit = cf_units.Unit('second') - if cube.coord('time').has_bounds(): + points = np.array( + time_delta.days * 24 * 60 * 60 + time_delta.seconds, dtype=np.int32 + ) + forecast_period_unit = cf_units.Unit("second") + if cube.coord("time").has_bounds(): time_window = time_coord.cell(0).bound - time_window = (time_window[1] - time_window[0]) - bounds = np.array([points - time_window.total_seconds(), points], - dtype=np.int32) + time_window = time_window[1] - time_window[0] + bounds = np.array( + [points - time_window.total_seconds(), points], dtype=np.int32 + ) else: bounds = None cube.add_aux_coord( - iris.coords.AuxCoord(points, standard_name='forecast_period', - bounds=bounds, units=forecast_period_unit)) + iris.coords.AuxCoord( + points, + standard_name="forecast_period", + bounds=bounds, + units=forecast_period_unit, + ) + ) def mask_cube(cube, field): @@ -373,15 +405,13 @@ def mask_cube(cube, field): if field.datum_type == 1: # field.data are integers if np.any(field.data == field.int_mdi): - cube.data = np.ma.masked_equal(field.data, - field.int_mdi) + cube.data = np.ma.masked_equal(field.data, field.int_mdi) elif field.datum_type == 0: # field.data are floats if np.any(np.isclose(field.data, field.float32_mdi)): cube.data = np.ma.masked_inside( - field.data, - field.float32_mdi - 0.5, - field.float32_mdi + 0.5) + field.data, field.float32_mdi - 0.5, field.float32_mdi + 0.5 + ) def experiment(cube, field): @@ -405,9 +435,14 @@ def proj_biaxial_ellipsoid(field): # Reference for airy_1830 and international_1924 ellipsoids: # http://fcm9/projects/PostProc/wiki/PostProcDocDomains#ProjectionConstants # Reference for GRS80: - airy_1830 = {'semi_major_axis': 6377563.396, 'semi_minor_axis': 6356256.910} - international_1924 = {'semi_major_axis': 6378388.000, - 'semi_minor_axis': 6356911.946} + airy_1830 = { + "semi_major_axis": 6377563.396, + "semi_minor_axis": 6356256.910, + } + international_1924 = { + "semi_major_axis": 6378388.000, + "semi_minor_axis": 6356911.946, + } if field.proj_biaxial_ellipsoid == 0: ellipsoid = airy_1830 elif field.proj_biaxial_ellipsoid == 1: @@ -415,16 +450,22 @@ def proj_biaxial_ellipsoid(field): elif is_missing(field, field.proj_biaxial_ellipsoid): if field.horizontal_grid_type == 0: ellipsoid = airy_1830 - elif field.horizontal_grid_type == 1 or field.horizontal_grid_type == 4: + elif ( + field.horizontal_grid_type == 1 or field.horizontal_grid_type == 4 + ): ellipsoid = international_1924 else: - raise TranslationError('''Unsupported grid type, only NG, EuroPP - and lat/long are possible''') + raise TranslationError( + """Unsupported grid type, only NG, EuroPP + and lat/long are possible""" + ) else: raise TranslationError( - 'Ellipsoid not supported, proj_biaxial_ellipsoid:{}, ' - 'horizontal_grid_type:{}'.format(field.proj_biaxial_ellipsoid, - field.horizontal_grid_type)) + "Ellipsoid not supported, proj_biaxial_ellipsoid:{}, " + "horizontal_grid_type:{}".format( + field.proj_biaxial_ellipsoid, field.horizontal_grid_type + ) + ) return ellipsoid @@ -434,17 +475,21 @@ def set_british_national_grid_defaults(field): files are missing these.""" if is_missing(field, field.true_origin_latitude): - field.true_origin_latitude = 49. + field.true_origin_latitude = 49.0 if is_missing(field, field.true_origin_longitude): - field.true_origin_longitude = -2. + field.true_origin_longitude = -2.0 if is_missing(field, field.true_origin_easting) or np.isclose( - # Some old files misquote the value in km instead of m - field.true_origin_easting, 400.): - field.true_origin_easting = 400000. + # Some old files misquote the value in km instead of m + field.true_origin_easting, + 400.0, + ): + field.true_origin_easting = 400000.0 if is_missing(field, field.true_origin_northing) or np.isclose( - # Some old files misquote the value in km instead of m - field.true_origin_northing, -100.): - field.true_origin_northing = -100000. + # Some old files misquote the value in km instead of m + field.true_origin_northing, + -100.0, + ): + field.true_origin_northing = -100000.0 if is_missing(field, field.tm_meridian_scaling): field.tm_meridian_scaling = 0.9996012717 @@ -466,9 +511,13 @@ def coord_system(field): set_british_national_grid_defaults(field) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: coord_sys = iris.coord_systems.TransverseMercator( - field.true_origin_latitude, field.true_origin_longitude, - field.true_origin_easting, field.true_origin_northing, - field.tm_meridian_scaling, iris.coord_systems.GeogCS(**ellipsoid)) + field.true_origin_latitude, + field.true_origin_longitude, + field.true_origin_easting, + field.true_origin_northing, + field.tm_meridian_scaling, + iris.coord_systems.GeogCS(**ellipsoid), + ) elif field.horizontal_grid_type == 1: coord_sys = iris.coord_systems.GeogCS(**ellipsoid) else: @@ -490,91 +539,107 @@ def horizontal_grid(cube, field): ) crs = coord_system(field) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: - units_name = 'm' - x_coord_name = 'projection_x_coordinate' - y_coord_name = 'projection_y_coordinate' + units_name = "m" + x_coord_name = "projection_x_coordinate" + y_coord_name = "projection_y_coordinate" elif field.horizontal_grid_type == 1: - units_name = 'degrees' - x_coord_name = 'longitude' - y_coord_name = 'latitude' + units_name = "degrees" + x_coord_name = "longitude" + y_coord_name = "latitude" else: - raise TranslationError("Horizontal grid type {} not " - "implemented".format(field.horizontal_grid_type)) - points = (np.arange(field.num_cols) * field.column_step + - field.x_origin).astype(np.float32) + raise TranslationError( + "Horizontal grid type {} not " + "implemented".format(field.horizontal_grid_type) + ) + points = ( + np.arange(field.num_cols) * field.column_step + field.x_origin + ).astype(np.float32) x_coord = DimCoord( - points, - standard_name=x_coord_name, - units=units_name, - coord_system=crs, + points, standard_name=x_coord_name, units=units_name, coord_system=crs, ) cube.add_dim_coord(x_coord, 1) - points = (np.arange(field.num_rows)[::-1] * -field.row_step + - field.y_origin).astype(np.float32) + points = ( + np.arange(field.num_rows)[::-1] * -field.row_step + field.y_origin + ).astype(np.float32) y_coord = DimCoord( - points, - standard_name=y_coord_name, - units=units_name, - coord_system=crs, + points, standard_name=y_coord_name, units=units_name, coord_system=crs, ) cube.add_dim_coord(y_coord, 0) def vertical_coord(cube, field): """Add a vertical coord to the cube, if appropriate.""" - if all([is_missing(field, x) for x in [ - field.vertical_coord, field.vertical_coord_type, - field.reference_vertical_coord, - field.reference_vertical_coord_type]]): + if all( + [ + is_missing(field, x) + for x in [ + field.vertical_coord, + field.vertical_coord_type, + field.reference_vertical_coord, + field.reference_vertical_coord_type, + ] + ] + ): return - if (not is_missing(field, field.reference_vertical_coord_type) and - field.reference_vertical_coord_type != field.vertical_coord_type - and not is_missing(field, field.reference_vertical_coord)): - msg = ('Unmatched vertical coord types ' - f'{field.vertical_coord_type} != ' - f'{field.reference_vertical_coord_type}' - f'. Assuming {field.vertical_coord_type}') + if ( + not is_missing(field, field.reference_vertical_coord_type) + and field.reference_vertical_coord_type != field.vertical_coord_type + and not is_missing(field, field.reference_vertical_coord) + ): + msg = ( + "Unmatched vertical coord types " + f"{field.vertical_coord_type} != " + f"{field.reference_vertical_coord_type}" + f". Assuming {field.vertical_coord_type}" + ) warnings.warn(msg) coord_point = field.vertical_coord - if coord_point == 8888.: + if coord_point == 8888.0: if "sea_level" not in cube.name(): cube.rename(f"{cube.name()}_at_mean_sea_level") - coord_point = 0. - if (np.isclose(field.reference_vertical_coord, 8888.) or - is_missing(field, field.reference_vertical_coord)): + coord_point = 0.0 + if np.isclose(field.reference_vertical_coord, 8888.0) or is_missing( + field, field.reference_vertical_coord + ): # This describes a surface field. No changes needed. return coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) - if np.isclose(coord_point, 9999.): - if (np.isclose(field.reference_vertical_coord, 9999.) or - is_missing(field, field.reference_vertical_coord)): + if np.isclose(coord_point, 9999.0): + if np.isclose(field.reference_vertical_coord, 9999.0) or is_missing( + field, field.reference_vertical_coord + ): # This describes a surface field. No changes needed. return # A bounded vertical coord starting from the surface - coord_point = 0. - coord_args = VERTICAL_CODES.get(field.reference_vertical_coord_type, - None) + coord_point = 0.0 + coord_args = VERTICAL_CODES.get( + field.reference_vertical_coord_type, None + ) coord_point = np.array(coord_point, dtype=np.float32) - if (field.reference_vertical_coord >= 0. and - field.reference_vertical_coord != coord_point): - bounds = np.array([coord_point, field.reference_vertical_coord], - dtype=np.float32) + if ( + field.reference_vertical_coord >= 0.0 + and field.reference_vertical_coord != coord_point + ): + bounds = np.array( + [coord_point, field.reference_vertical_coord], dtype=np.float32 + ) else: bounds = None if coord_args: - new_coord = iris.coords.AuxCoord(coord_point, bounds=bounds, - **coord_args) + new_coord = iris.coords.AuxCoord( + coord_point, bounds=bounds, **coord_args + ) # Add coordinate to cube cube.add_aux_coord(new_coord) return warnings.warn( - "Vertical coord {!r} not yet handled" "".format( - field.vertical_coord_type), + "Vertical coord {!r} not yet handled" + "".format(field.vertical_coord_type), TranslationWarning, ) @@ -583,9 +648,11 @@ def ensemble_member(cube, field): """Add an 'ensemble member' coord to the cube, if present in the field.""" ensemble_member_value = getattr(field, "ensemble_member") if not is_missing(field, ensemble_member_value): - cube.add_aux_coord(DimCoord(np.array(ensemble_member_value, - dtype=np.int32), - "realization")) + cube.add_aux_coord( + DimCoord( + np.array(ensemble_member_value, dtype=np.int32), "realization" + ) + ) def origin_corner(cube, field): @@ -608,8 +675,8 @@ def add_attr(item): value = getattr(field, item) if is_missing(field, value): return - if 'radius' in item: - value = f'{value} km' + if "radius" in item: + value = f"{value} km" cube.attributes[item] = value add_attr("nimrod_version") @@ -633,20 +700,23 @@ def add_attr(item): add_attr("sensor_id") add_attr("meteosat_id") add_attr("alphas_available") - if 'radar' not in field.source: - for key in ["neighbourhood_radius", "recursive_filter_iterations", - "recursive_filter_alpha", "threshold_vicinity_radius", - "probability_period_of_event"]: + if "radar" not in field.source: + for key in [ + "neighbourhood_radius", + "recursive_filter_iterations", + "recursive_filter_alpha", + "threshold_vicinity_radius", + "probability_period_of_event", + ]: add_attr(key) source = field.source.strip() - rematcher = re.compile('^ek\d\d$') - if (rematcher.match(source) is not None - or source.find('umek') == 0): - source = 'MOGREPS-UK' - cube.attributes['source'] = source - cube.attributes['title'] = 'Unknown' - cube.attributes['institution'] = 'Met Office' + rematcher = re.compile("^ek\d\d$") + if rematcher.match(source) is not None or source.find("umek") == 0: + source = "MOGREPS-UK" + cube.attributes["source"] = source + cube.attributes["title"] = "Unknown" + cube.attributes["institution"] = "Met Office" def known_threshold_coord(field): @@ -654,32 +724,39 @@ def known_threshold_coord(field): Supplies known threshold coord meta-data for known use cases. """ coord_keys = {} - if field.field_code == 161 and field.threshold_value >= 0.: + if field.field_code == 161 and field.threshold_value >= 0.0: coord_keys = {"var_name": "threshold"} - if field.threshold_value_alt > 8.: + if field.threshold_value_alt > 8.0: coord_keys["standard_name"] = "height" coord_keys["units"] = "metres" else: coord_keys["standard_name"] = "cloud_area_fraction" coord_keys["units"] = "oktas" - if field.field_code == 29 and field.threshold_value >= 0.: + if field.field_code == 29 and field.threshold_value >= 0.0: if is_missing(field, field.threshold_type): - coord_keys = {"standard_name": "visibility_in_air", - "var_name": "threshold", - "units": "metres"} + coord_keys = { + "standard_name": "visibility_in_air", + "var_name": "threshold", + "units": "metres", + } else: - coord_keys = {"long_name": "fog_fraction", - "var_name": "threshold", - "units": "1"} - if (field.field_code == 422 - and field.threshold_value >= 0. - and is_missing(field, field.threshold_type)): - coord_keys = {"long_name": "radius_of_max", - "units": "km"} + coord_keys = { + "long_name": "fog_fraction", + "var_name": "threshold", + "units": "1", + } + if ( + field.field_code == 422 + and field.threshold_value >= 0.0 + and is_missing(field, field.threshold_type) + ): + coord_keys = {"long_name": "radius_of_max", "units": "km"} if field.field_code == 821: - coord_keys = {"standard_name": "wind_speed_of_gust", - "var_name": "threshold", - "units": "m/s"} + coord_keys = { + "standard_name": "wind_speed_of_gust", + "var_name": "threshold", + "units": "m/s", + } return coord_keys @@ -689,59 +766,73 @@ def probability_coord(cube, field): cube if appropriate. Returns True if this is a blended multi-member field """ - probtype_lookup = {1: {'var_name': 'threshold', - 'attributes': {'relative_to_threshold': 'above'}}, - 2: {'var_name': 'threshold', - 'attributes': {'relative_to_threshold': 'below'}}, - 3: {'long_name': 'percentile', 'units': "1"}, - 4: {'var_name': 'threshold', - 'attributes': {'relative_to_threshold': 'equal'}}} - probmethod_lookup = {1: 'AOT (Any One Time)', - 2: 'ST (Some Time)', - 4: 'AT (All Time)', - 8: 'AOL (Any One Location)', - 16: 'SW (Some Where)'} + probtype_lookup = { + 1: { + "var_name": "threshold", + "attributes": {"relative_to_threshold": "above"}, + }, + 2: { + "var_name": "threshold", + "attributes": {"relative_to_threshold": "below"}, + }, + 3: {"long_name": "percentile", "units": "1"}, + 4: { + "var_name": "threshold", + "attributes": {"relative_to_threshold": "equal"}, + }, + } + probmethod_lookup = { + 1: "AOT (Any One Time)", + 2: "ST (Some Time)", + 4: "AT (All Time)", + 8: "AOL (Any One Location)", + 16: "SW (Some Where)", + } is_multi_member_field = False coord_keys = probtype_lookup.get(field.threshold_type, {}) if coord_keys: is_multi_member_field = True coord_keys.update(known_threshold_coord(field)) - if not coord_keys.get('units', None): - coord_keys['units'] = DEFAULT_UNITS.get(field.field_code, None) + if not coord_keys.get("units", None): + coord_keys["units"] = DEFAULT_UNITS.get(field.field_code, None) coord_val = None - if field.threshold_value_alt > -32766.: + if field.threshold_value_alt > -32766.0: coord_val = field.threshold_value_alt - elif field.threshold_value > -32766.: + elif field.threshold_value > -32766.0: coord_val = field.threshold_value - if field.title.find('pc') > 0: + if field.title.find("pc") > 0: try: - coord_val = [int(x.strip('pc')) for x in field.title.split(' ') - if x.find('pc') > 0][0] + coord_val = [ + int(x.strip("pc")) + for x in field.title.split(" ") + if x.find("pc") > 0 + ][0] except IndexError: pass if coord_val is not None: - if field.threshold_fuzziness > -32766.: - bounds = [coord_val * field.threshold_fuzziness, - coord_val * (2. - field.threshold_fuzziness)] + if field.threshold_fuzziness > -32766.0: + bounds = [ + coord_val * field.threshold_fuzziness, + coord_val * (2.0 - field.threshold_fuzziness), + ] bounds = np.array(bounds, dtype=np.float32) else: bounds = None - if coord_keys.get('units', None) == 'oktas': - coord_keys['units'] = '1' - coord_val /= 8. + if coord_keys.get("units", None) == "oktas": + coord_keys["units"] = "1" + coord_val /= 8.0 if bounds is not None: - bounds /= 8. + bounds /= 8.0 new_coord = iris.coords.AuxCoord( - np.array(coord_val, dtype=np.float32), - bounds=bounds, - **coord_keys) + np.array(coord_val, dtype=np.float32), bounds=bounds, **coord_keys + ) cube.add_aux_coord(new_coord) if field.threshold_type == 3: pass else: if is_multi_member_field: - cube.units = '1' - cube.rename(f'probability_of_{cube.name()}') + cube.units = "1" + cube.rename(f"probability_of_{cube.name()}") if field.probability_method > 0: probability_attributes = [] @@ -750,7 +841,7 @@ def probability_coord(cube, field): if num >= key: probability_attributes.append(probmethod_lookup[key]) num = num - key - cube.attributes['Probability methods'] = probability_attributes + cube.attributes["Probability methods"] = probability_attributes if field.member_count == 1: is_multi_member_field = False return is_multi_member_field @@ -761,9 +852,11 @@ def soil_type_coord(cube, field): if field.threshold_type != 0: soil_name = SOIL_TYPE_CODES.get(field.soil_type, None) if soil_name: - cube.add_aux_coord(iris.coords.AuxCoord( - soil_name, - standard_name='soil_type', units=None)) + cube.add_aux_coord( + iris.coords.AuxCoord( + soil_name, standard_name="soil_type", units=None + ) + ) def time_averaging(cube, field): @@ -775,7 +868,7 @@ def time_averaging(cube, field): averaging_attributes.append(TIME_AVERAGING_CODES[key]) num = num - key if averaging_attributes: - cube.attributes['processing'] = averaging_attributes + cube.attributes["processing"] = averaging_attributes def run(field): diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index d165c46ca0..702f4e06ce 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -40,45 +40,39 @@ def test_multi_field_load(self): @tests.skip_data def test_huge_field_load(self): # load a wide range of cubes with all meta-data variations - for datafile in {"u1096_ng_ek07_precip0540_accum180_18km", - "u1096_ng_ek00_cloud3d0060_2km", - "u1096_ng_ek00_cloud_2km", - "u1096_ng_ek00_convection_2km", - "u1096_ng_ek00_convwind_2km", - "u1096_ng_ek00_frzlev_2km", - "u1096_ng_ek00_height_2km", - "u1096_ng_ek00_precip_2km", - "u1096_ng_ek00_precipaccum_2km", - "u1096_ng_ek00_preciptype_2km", - "u1096_ng_ek00_pressure_2km", - "u1096_ng_ek00_radiation_2km", - "u1096_ng_ek00_radiationuv_2km", - "u1096_ng_ek00_refl_2km", - "u1096_ng_ek00_relhumidity3d0060_2km", - "u1096_ng_ek00_relhumidity_2km", - "u1096_ng_ek00_snow_2km", - "u1096_ng_ek00_soil3d0060_2km", - "u1096_ng_ek00_soil_2km", - "u1096_ng_ek00_temperature_2km", - "u1096_ng_ek00_visibility_2km", - "u1096_ng_ek00_wind_2km", - "u1096_ng_ek00_winduv3d0015_2km", - "u1096_ng_ek00_winduv_2km", - "u1096_ng_ek01_cape_2km", - "u1096_ng_umqv_fog_2km", - "u1096_ng_bmr04_precip_2km", - "u1096_ng_bsr05_precip_accum60_2km", - "probability_fields", - }: + for datafile in { + "u1096_ng_ek07_precip0540_accum180_18km", + "u1096_ng_ek00_cloud3d0060_2km", + "u1096_ng_ek00_cloud_2km", + "u1096_ng_ek00_convection_2km", + "u1096_ng_ek00_convwind_2km", + "u1096_ng_ek00_frzlev_2km", + "u1096_ng_ek00_height_2km", + "u1096_ng_ek00_precip_2km", + "u1096_ng_ek00_precipaccum_2km", + "u1096_ng_ek00_preciptype_2km", + "u1096_ng_ek00_pressure_2km", + "u1096_ng_ek00_radiation_2km", + "u1096_ng_ek00_radiationuv_2km", + "u1096_ng_ek00_refl_2km", + "u1096_ng_ek00_relhumidity3d0060_2km", + "u1096_ng_ek00_relhumidity_2km", + "u1096_ng_ek00_snow_2km", + "u1096_ng_ek00_soil3d0060_2km", + "u1096_ng_ek00_soil_2km", + "u1096_ng_ek00_temperature_2km", + "u1096_ng_ek00_visibility_2km", + "u1096_ng_ek00_wind_2km", + "u1096_ng_ek00_winduv3d0015_2km", + "u1096_ng_ek00_winduv_2km", + "u1096_ng_ek01_cape_2km", + "u1096_ng_umqv_fog_2km", + "u1096_ng_bmr04_precip_2km", + "u1096_ng_bsr05_precip_accum60_2km", + "probability_fields", + }: cube = iris.load( - tests.get_data_path( - ( - "NIMROD", - "uk2km", - "cutouts", - datafile, - ) - ) + tests.get_data_path(("NIMROD", "uk2km", "cutouts", datafile,)) ) self.assertCML(cube, ("nimrod", f"{datafile}.cml")) diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py index 48f03049d1..226eb3fa61 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_vertical_coord.py @@ -38,10 +38,13 @@ def setUp(self): ) self.cube = mock.Mock() - def _call_vertical_coord(self, vertical_coord_val=None, - vertical_coord_type=None, - reference_vertical_coord=None, - reference_vertical_coord_type=None): + def _call_vertical_coord( + self, + vertical_coord_val=None, + vertical_coord_type=None, + reference_vertical_coord=None, + reference_vertical_coord_type=None, + ): if vertical_coord_val: self.field.vertical_coord = vertical_coord_val if vertical_coord_type: @@ -49,14 +52,16 @@ def _call_vertical_coord(self, vertical_coord_val=None, if reference_vertical_coord: self.field.reference_vertical_coord = reference_vertical_coord if reference_vertical_coord_type: - self.field.reference_vertical_coord_type = \ + self.field.reference_vertical_coord_type = ( reference_vertical_coord_type + ) vertical_coord(self.cube, self.field) def test_unhandled(self): with mock.patch("warnings.warn") as warn: - self._call_vertical_coord(vertical_coord_val=1., - vertical_coord_type=-1) + self._call_vertical_coord( + vertical_coord_val=1.0, vertical_coord_type=-1 + ) warn.assert_called_once_with( "Vertical coord -1 not yet handled", TranslationWarning ) @@ -69,8 +74,9 @@ def test_null(self): def test_ground_level(self): with mock.patch("warnings.warn") as warn: - self._call_vertical_coord(vertical_coord_val=9999., - vertical_coord_type=0) + self._call_vertical_coord( + vertical_coord_val=9999.0, vertical_coord_type=0 + ) self.assertEqual(warn.call_count, 0) From ed451e587cc559fceec18ef37fc774ac53d01986 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 6 Mar 2020 13:38:58 +0000 Subject: [PATCH 18/46] As the crs origin information in some old Nimrod files has been corrected, these test results need updating as they were NOT based on an accurate OSGB grid. --- lib/iris/tests/integration/test_regridding.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/integration/test_regridding.py b/lib/iris/tests/integration/test_regridding.py index 0a6a78f2df..42cd2dc0b0 100644 --- a/lib/iris/tests/integration/test_regridding.py +++ b/lib/iris/tests/integration/test_regridding.py @@ -59,12 +59,11 @@ def _regrid(self, method): def test_linear(self): res = self._regrid("linear") - self.assertArrayShapeStats(res, (73, 96), -15539.099752, 5825.651452) + self.assertArrayShapeStats(res, (73, 96), 17799.296120, 11207.701323) def test_nearest(self): res = self._regrid("nearest") - self.assertArrayShapeStats(res, (73, 96), -15542.226501, 5839.922010) - + self.assertArrayShapeStats(res, (73, 96), 17808.068828, 11225.314310) @tests.skip_data class TestGlobalSubsample(tests.IrisTest): From f5e22e6ed166faa1c05198d72d80b46e3bf7df0c Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 6 Mar 2020 13:40:59 +0000 Subject: [PATCH 19/46] Improves regex syntax to make Stickler happy. --- lib/iris/fileformats/nimrod_load_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index a6be41fb56..33024c331e 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -711,7 +711,7 @@ def add_attr(item): add_attr(key) source = field.source.strip() - rematcher = re.compile("^ek\d\d$") + rematcher = re.compile(r"^ek\d\d$") if rematcher.match(source) is not None or source.find("umek") == 0: source = "MOGREPS-UK" cube.attributes["source"] = source From 2aca4c5eab8e001891c831d18d6cd49c0d8cbe3a Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 6 Mar 2020 13:56:33 +0000 Subject: [PATCH 20/46] Adds missing blank line for black. --- lib/iris/tests/integration/test_regridding.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/iris/tests/integration/test_regridding.py b/lib/iris/tests/integration/test_regridding.py index 42cd2dc0b0..4375e3b45a 100644 --- a/lib/iris/tests/integration/test_regridding.py +++ b/lib/iris/tests/integration/test_regridding.py @@ -65,6 +65,7 @@ def test_nearest(self): res = self._regrid("nearest") self.assertArrayShapeStats(res, (73, 96), 17808.068828, 11225.314310) + @tests.skip_data class TestGlobalSubsample(tests.IrisTest): def setUp(self): From 0ce9af16b31fa1e235d8fc58c53c70bbd194c76a Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 23 Apr 2020 12:59:58 +0100 Subject: [PATCH 21/46] Updates result for broken test Following earlier change to known inaccuracies in Nimrod-format meta-data (projection true-origin is 10*3 out in some data), one test result had been overlooked. --- lib/iris/tests/results/nimrod/load_2flds.cml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/iris/tests/results/nimrod/load_2flds.cml b/lib/iris/tests/results/nimrod/load_2flds.cml index 72f06f8cd1..b068657d40 100644 --- a/lib/iris/tests/results/nimrod/load_2flds.cml +++ b/lib/iris/tests/results/nimrod/load_2flds.cml @@ -17,15 +17,15 @@ - - + - - + From 8c365555a00a366e6ca1e3b99221774b0db0bb28 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 23 Apr 2020 13:01:25 +0100 Subject: [PATCH 22/46] Reorders how units are handled following a bug investigation Masking of data array was undoing the unit-conversion logic, so I have reversed the order of these. I have also added tests for the unit-conversion as I have written them and they work. --- lib/iris/fileformats/nimrod_load_rules.py | 7 +- .../nimrod_load_rules/test_units.py | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 33024c331e..6d580b4633 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -405,12 +405,12 @@ def mask_cube(cube, field): if field.datum_type == 1: # field.data are integers if np.any(field.data == field.int_mdi): - cube.data = np.ma.masked_equal(field.data, field.int_mdi) + cube.data = np.ma.masked_equal(cube.data, field.int_mdi) elif field.datum_type == 0: # field.data are floats if np.any(np.isclose(field.data, field.float32_mdi)): cube.data = np.ma.masked_inside( - field.data, field.float32_mdi - 0.5, field.float32_mdi + 0.5 + cube.data, field.float32_mdi - 0.5, field.float32_mdi + 0.5 ) @@ -887,9 +887,8 @@ def run(field): cube = iris.cube.Cube(field.data.astype(np.float32)) name(cube, field) - units(cube, field) - mask_cube(cube, field) + units(cube, field) # time time(cube, field) diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py new file mode 100644 index 0000000000..85792589b3 --- /dev/null +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py @@ -0,0 +1,72 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the `iris.fileformats.nimrod_load_rules.units` function. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +from unittest import mock +import numpy as np + +from iris.cube import Cube +from iris.fileformats.nimrod_load_rules import ( + units, + NIMROD_DEFAULT, + TranslationWarning, +) +from iris.fileformats.nimrod import NimrodField + + +class Test(tests.IrisTest): + NIMROD_LOCATION = "iris.fileformats.nimrod_load_rules" + + def setUp(self): + self.field = mock.Mock( + units="", + int_mdi=-32767, + float32_mdi=NIMROD_DEFAULT, + spec=NimrodField, + ) + self.cube = mock.Mock( + data=np.zeros((3, 3), dtype=np.float32), spec=Cube + ) + + def _call_units( + self, data=None, units_str=None, + ): + if data is not None: + self.cube.data = data + if units_str: + self.field.units = units_str + units(self.cube, self.field) + + def test_null(self): + with mock.patch("warnings.warn") as warn: + self._call_units(data=np.ones_like(self.cube.data), units_str="m") + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "m") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + + def test_times32(self): + with mock.patch("warnings.warn") as warn: + self._call_units( + data=np.ones_like(self.cube.data) * 32, units_str="mm/hr*32" + ) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "mm/hr") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + + +if __name__ == "__main__": + tests.main() From 79e5d6e144e2cd9089280fdefedb282e7928eb44 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 23 Apr 2020 13:47:46 +0100 Subject: [PATCH 23/46] Removes unused import --- lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py index 85792589b3..019f523a1c 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py @@ -19,7 +19,6 @@ from iris.fileformats.nimrod_load_rules import ( units, NIMROD_DEFAULT, - TranslationWarning, ) from iris.fileformats.nimrod import NimrodField From f97d4e006a0279a47677ca53678d2633d8c93056 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Wed, 13 May 2020 16:33:45 +0100 Subject: [PATCH 24/46] Corrects data checksum following unit conversion fix. Points to latest version of test data --- .travis.yml | 2 +- lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b51a18d1d..bc9c86491c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ git: install: - > - export IRIS_TEST_DATA_REF="06e3c7d289d4ae177539e71134a2a6f6a1bc8117"; + export IRIS_TEST_DATA_REF="22166475999c4ba0f106b9214b71b667b8aee016"; export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//"); # Install miniconda diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml index d79dd83f55..8260ce8ee6 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml @@ -37,7 +37,7 @@ - + @@ -117,6 +117,6 @@ - + From 57fd2adc307b8dc8b2174bc34aab79b7b81a4652 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 18 May 2020 09:12:48 +0100 Subject: [PATCH 25/46] Updates test-data reference following merging of nimrod-data PR. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bc9c86491c..312914d634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ git: install: - > - export IRIS_TEST_DATA_REF="22166475999c4ba0f106b9214b71b667b8aee016"; + export IRIS_TEST_DATA_REF="fffb9b14b9cb472c5eb2ebb7fd19acb7f6414a30"; export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//"); # Install miniconda From 674c828a58d6c07e7343320c991f9d0f83765407 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 18 May 2020 12:33:38 +0100 Subject: [PATCH 26/46] Responds to first review - Quite a lot of minor refactoring to make the code clearer - One unit-test has an extra input - One probability field has changed behaviour. Probability of freezing rain (precipitation_type 40) now has units of "1" for the categorical threshold coord. --- lib/iris/fileformats/nimrod_load_rules.py | 397 +++++++++--------- .../results/nimrod/probability_fields.cml | 2 +- lib/iris/tests/test_nimrod.py | 1 + 3 files changed, 206 insertions(+), 194 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 6d580b4633..203e5512c8 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -32,88 +32,6 @@ ) -DEFAULT_UNITS = { - 817: "m s^-1", - 804: "knots", - 422: "min^-1", - 218: "cm", - 172: "oktas", - 155: "m", - 101: "m", - 63: "mm hr^-1", - 61: "mm", - 58: "Celsius", - 12: "mb", - 8: "1", -} -FIELD_CODES = {73: "orography"} -# VERTICAL_CODES contains conversions from the Nimrod Documentation for the -# header entry 20 for the vertical coordinate type -VERTICAL_CODES = { - 0: { - "standard_name": "height", - "units": "m", - "attributes": {"positive": "up"}, - }, - 1: { - "standard_name": "altitude", - "units": "m", - "attributes": {"positive": "up"}, - }, - 2: { - "standard_name": "air_pressure", - "units": "hPa", - "attributes": {"positive": "down"}, - }, - 6: {"standard_name": "air_temperature", "units": "K"}, - 12: { - "long_name": "depth_below_ground", - "units": "m", - "attributes": {"positive": "down"}, - }, -} -# Unhandled VERTICAL_CODES values (no use case identified): -# 3: ['sigma', 'model level'], -# 4: ['eta', 'model level'], -# 5: ['radar beam number', 'unknown'], -# 7: ['potential temperature', 'unknown'], -# 8: ['equivalent potential temperature', 'unknown'], -# 9: ['wet bulb potential temperature', 'unknown'], -# 10: ['potential vorticity', 'unknown'], -# 11: ['cloud boundary', 'unknown'], - -SOIL_TYPE_CODES = { - 1: "broadleaf_tree", - 2: "needleleaf_tree", - 3: "c3_grass", - 4: "c4_grass", - 5: "crop", - 6: "shrub", - 7: "urban", - 8: "water", - 9: "soil", - 10: "ice", - 601: "urban_canyon", - 602: "urban_roof", -} -TIME_AVERAGING_CODES = { - 8192: "maximum in period", - 4096: "minimum in period", - 2048: "unknown(2048)", - 1024: "unknown(1024)", - 512: "time lagged", - 256: "extrapolation", - 128: "accumulation or average", - 64: "from UM 150m", - 32: "scaled to UM resolution", - 16: "averaged over multiple surface types", - 8: "only observations used", - 4: "smoothed", - 2: "cold bias applied", - 1: "warm bias applied", -} - - class TranslationWarning(Warning): pass @@ -130,67 +48,55 @@ def name(cube, field): Modifies the Nimrod object title based on other meta-data in the Nimrod field and known use cases. """ - if field.field_code == 12: - field.title = "air_pressure" - if field.field_code == 27: - field.title = "snow fraction" - if field.field_code == 28: - field.title = "snow probability" - if field.field_code == 29 and field.threshold_value >= 0.0: - field.title = "fog fraction" - if field.field_code == 58: - field.title = "temperature" - if field.field_code == 61: - field.title = "precipitation" - if field.field_code == 63: - field.title = "precipitation" - if field.field_code == 101: - field.title = "snow_melting_level_above_sea_level" - if field.field_code == 102: - field.title = "rain_melted_level_above_sea_level" - if field.field_code == 155: - field.title = "Visibility" - if field.field_code == 156: - field.title = "Worst visibility in grid point" - if field.field_code == 161 and field.threshold_value >= 0.0: - field.title = "minimum_cloud_base_above_threshold" - if field.field_code == 218: - field.title = "snowfall" - if field.field_code == 172: - field.title = "cloud_area_fraction_in_atmosphere" - if field.field_code == 421: - field.title = "precipitation type" - if field.field_code == 501: - field.title = "vector_wind_shear" - field.source = "" - if field.field_code == 508: - field.title = "low_level_jet_u_component" - if field.field_code == 509: - field.title = "low_level_jet_curvature" - if field.field_code == 514: - field.title = "low_level_jet_v_component" - if field.field_code == 804 and field.vertical_coord >= 0.0: - field.title = "wind speed" - if field.field_code == 806 and field.vertical_coord >= 0.0: - field.title = "wind direction" - if field.field_code == 817: - field.title = "wind_speed_of_gust" - if field.field_code == 821: - field.title = "Probabilistic Gust Risk Analysis from Observations" - field.source = "Nimrod pwind routine" - if field.source.strip() == "pwind": - field.source = "Nimrod pwind routine" + title_from_field_code = { + 12: "air_pressure", + 27: "snow fraction", + 28: "snow probability", + 58: "temperature", + 61: "precipitation", + 63: "precipitation", + 101: "snow_melting_level_above_sea_level", + 102: "rain_melted_level_above_sea_level", + 155: "Visibility", + 156: "Worst visibility in grid point", + 172: "cloud_area_fraction_in_atmosphere", + 218: "snowfall", + 421: "precipitation type", + 501: "vector_wind_shear", + 508: "low_level_jet_u_component", + 509: "low_level_jet_curvature", + 514: "low_level_jet_v_component", + 817: "wind_speed_of_gust", + 821: "Probabilistic Gust Risk Analysis from Observations", + } + # threshold_value >= 0 + title_from_field_code_and_threshold = { + 29: "fog fraction", + 161: "minimum_cloud_base_above_threshold", + } + # vertical_coord >= 0 + title_from_field_code_and_vcoord = { + 804: "wind speed", + 806: "wind direction", + } + cube_title = title_from_field_code.get(field.field_code, field.title) + if field.threshold_value >= 0: + cube_title = title_from_field_code_and_threshold.get( + field.field_code, cube_title + ) + if field.vertical_coord >= 0: + cube_title = title_from_field_code_and_vcoord.get( + field.field_code, cube_title + ) - if getattr(field, "ensemble_member") == -98: - if "mean" not in field.title: - field.title = "mean_of_" + field.title - field.ensemble_member = field.int_mdi - if getattr(field, "ensemble_member") == -99: - if "spread" not in field.title: - field.title = "standard_deviation_of_" + field.title - field.ensemble_member = field.int_mdi + if field.ensemble_member == -98: + if "mean" not in cube_title: + cube_title = "mean_of_" + cube_title + if field.ensemble_member == -99: + if "spread" not in cube_title: + cube_title = "standard_deviation_of_" + cube_title - cube.rename(remove_unprintable_chars(field.title)) + cube.rename(remove_unprintable_chars(cube_title)) def remove_unprintable_chars(input_str): @@ -256,12 +162,10 @@ def units(cube, field): unit_list = field_units.split("E") cube.data = cube.data / 10.0 ** float(unit_list[1]) field_units = unit_list[0] - if "%" in field_units: + if field_units == "%": # Convert any percentages into fraction - unit_list = field_units.split("%") - if len("".join(unit_list)) == 0: - field_units = "1" - cube.data = cube.data / 100.0 + field_units = "1" + cube.data = cube.data / 100.0 if field_units == "oktas": field_units = "1" cube.data /= 8.0 @@ -298,13 +202,8 @@ def time(cube, field): """Add a time coord to the cube.""" if field.vt_year <= 0: # Some ancillary files, eg land sea mask do not - # have a validity time. So make one up for the - # start of the year. - # This will screw up the forecast_period for these fields, - # although if the valid time is missing too, it will be - # made to be the same, so the forecast_period will always - # be zero for these files. - valid_date = cftime.datetime(2016, 1, 1, 0, 0, 0) + # have a validity time. + return else: valid_date = cftime.datetime( field.vt_year, @@ -401,17 +300,15 @@ def mask_cube(cube, field): Updates cube.data to be a masked array if appropriate. """ - + masked_points = None if field.datum_type == 1: # field.data are integers - if np.any(field.data == field.int_mdi): - cube.data = np.ma.masked_equal(cube.data, field.int_mdi) + masked_points = field.data == field.int_mdi elif field.datum_type == 0: # field.data are floats - if np.any(np.isclose(field.data, field.float32_mdi)): - cube.data = np.ma.masked_inside( - cube.data, field.float32_mdi - 0.5, field.float32_mdi + 0.5 - ) + masked_points = np.isclose(field.data, field.float32_mdi) + if np.any(masked_points): + cube.data = np.ma.masked_array(cube.data, mask=masked_points) def experiment(cube, field): @@ -551,16 +448,24 @@ def horizontal_grid(cube, field): "Horizontal grid type {} not " "implemented".format(field.horizontal_grid_type) ) - points = ( - np.arange(field.num_cols) * field.column_step + field.x_origin - ).astype(np.float32) + points = np.linspace( + field.x_origin, + field.x_origin + field.num_cols * field.column_step, + field.num_cols, + endpoint=False, + dtype=np.float32, + ) x_coord = DimCoord( points, standard_name=x_coord_name, units=units_name, coord_system=crs, ) cube.add_dim_coord(x_coord, 1) - points = ( - np.arange(field.num_rows)[::-1] * -field.row_step + field.y_origin - ).astype(np.float32) + points = np.linspace( + field.y_origin - (field.num_rows-1) * field.row_step, + field.y_origin, + field.num_rows, + endpoint=True, + dtype=np.float32, + ) y_coord = DimCoord( points, standard_name=y_coord_name, units=units_name, coord_system=crs, ) @@ -569,6 +474,40 @@ def horizontal_grid(cube, field): def vertical_coord(cube, field): """Add a vertical coord to the cube, if appropriate.""" + # vertical_codes contains conversions from the Nimrod Documentation for the + # header entry 20 for the vertical coordinate type + # Unhandled vertical_codes values (no use case identified): + # 3: ['sigma', 'model level'], + # 4: ['eta', 'model level'], + # 5: ['radar beam number', 'unknown'], + # 7: ['potential temperature', 'unknown'], + # 8: ['equivalent potential temperature', 'unknown'], + # 9: ['wet bulb potential temperature', 'unknown'], + # 10: ['potential vorticity', 'unknown'], + # 11: ['cloud boundary', 'unknown'], + vertical_codes = { + 0: { + "standard_name": "height", + "units": "m", + "attributes": {"positive": "up"}, + }, + 1: { + "standard_name": "altitude", + "units": "m", + "attributes": {"positive": "up"}, + }, + 2: { + "standard_name": "air_pressure", + "units": "hPa", + "attributes": {"positive": "down"}, + }, + 6: {"standard_name": "air_temperature", "units": "K"}, + 12: { + "long_name": "depth_below_ground", + "units": "m", + "attributes": {"positive": "down"}, + }, + } if all( [ is_missing(field, x) @@ -606,7 +545,7 @@ def vertical_coord(cube, field): # This describes a surface field. No changes needed. return - coord_args = VERTICAL_CODES.get(field.vertical_coord_type, None) + coord_args = vertical_codes.get(field.vertical_coord_type, None) if np.isclose(coord_point, 9999.0): if np.isclose(field.reference_vertical_coord, 9999.0) or is_missing( field, field.reference_vertical_coord @@ -615,7 +554,7 @@ def vertical_coord(cube, field): return # A bounded vertical coord starting from the surface coord_point = 0.0 - coord_args = VERTICAL_CODES.get( + coord_args = vertical_codes.get( field.reference_vertical_coord_type, None ) coord_point = np.array(coord_point, dtype=np.float32) @@ -646,7 +585,11 @@ def vertical_coord(cube, field): def ensemble_member(cube, field): """Add an 'ensemble member' coord to the cube, if present in the field.""" - ensemble_member_value = getattr(field, "ensemble_member") + ensemble_member_value = field.ensemble_member + + if ensemble_member_value in [-98, -99]: + # ignore these special values handled in name() + return if not is_missing(field, ensemble_member_value): cube.add_aux_coord( DimCoord( @@ -700,7 +643,16 @@ def add_attr(item): add_attr("sensor_id") add_attr("meteosat_id") add_attr("alphas_available") - if "radar" not in field.source: + cube_source = field.source.strip() + + # Handle a few known meta-data errors: + if field.field_code == 501: + cube_source = "" + if field.field_code == 821: + cube_source = "Nimrod pwind routine" + if field.source.strip() == "pwind": + cube_source = "Nimrod pwind routine" + if "radar" not in cube_source: for key in [ "neighbourhood_radius", "recursive_filter_iterations", @@ -710,11 +662,13 @@ def add_attr(item): ]: add_attr(key) - source = field.source.strip() rematcher = re.compile(r"^ek\d\d$") - if rematcher.match(source) is not None or source.find("umek") == 0: - source = "MOGREPS-UK" - cube.attributes["source"] = source + if ( + rematcher.match(cube_source) is not None + or cube_source.find("umek") == 0 + ): + cube_source = "MOGREPS-UK" + cube.attributes["source"] = cube_source cube.attributes["title"] = "Unknown" cube.attributes["institution"] = "Met Office" @@ -732,7 +686,7 @@ def known_threshold_coord(field): else: coord_keys["standard_name"] = "cloud_area_fraction" coord_keys["units"] = "oktas" - if field.field_code == 29 and field.threshold_value >= 0.0: + elif field.field_code == 29 and field.threshold_value >= 0.0: if is_missing(field, field.threshold_type): coord_keys = { "standard_name": "visibility_in_air", @@ -745,13 +699,13 @@ def known_threshold_coord(field): "var_name": "threshold", "units": "1", } - if ( + elif ( field.field_code == 422 and field.threshold_value >= 0.0 and is_missing(field, field.threshold_type) ): coord_keys = {"long_name": "radius_of_max", "units": "km"} - if field.field_code == 821: + elif field.field_code == 821: coord_keys = { "standard_name": "wind_speed_of_gust", "var_name": "threshold", @@ -764,7 +718,7 @@ def probability_coord(cube, field): """ Adds a coord relating to probability meta-data from the header to the cube if appropriate. - Returns True if this is a blended multi-member field + Must be run after the name method. """ probtype_lookup = { 1: { @@ -788,23 +742,42 @@ def probability_coord(cube, field): 8: "AOL (Any One Location)", 16: "SW (Some Where)", } - is_multi_member_field = False + # The units for the threshold coord are not defined in the Nimrod meta-data. + # These represent the known use-cases. + units_from_field_code = { + 817: "m s^-1", + 804: "knots", + 422: "min^-1", + 421: "1", + 218: "cm", + 172: "oktas", + 155: "m", + 101: "m", + 63: "mm hr^-1", + 61: "mm", + 58: "Celsius", + 12: "mb", + 8: "1", + } + is_probability_field = False coord_keys = probtype_lookup.get(field.threshold_type, {}) if coord_keys: - is_multi_member_field = True + is_probability_field = True coord_keys.update(known_threshold_coord(field)) - if not coord_keys.get("units", None): - coord_keys["units"] = DEFAULT_UNITS.get(field.field_code, None) + if not coord_keys.get("units"): + coord_keys["units"] = units_from_field_code.get( + field.field_code, "unknown" + ) coord_val = None if field.threshold_value_alt > -32766.0: coord_val = field.threshold_value_alt elif field.threshold_value > -32766.0: coord_val = field.threshold_value - if field.title.find("pc") > 0: + if cube.name().find("pc") > 0: try: coord_val = [ int(x.strip("pc")) - for x in field.title.split(" ") + for x in cube.name().split(" ") if x.find("pc") > 0 ][0] except IndexError: @@ -823,6 +796,15 @@ def probability_coord(cube, field): coord_val /= 8.0 if bounds is not None: bounds /= 8.0 + if coord_keys["units"] == "unknown": + coord_name = coord_keys.get( + "standard_name", + coord_keys.get("long_name", coord_keys.get("var_name", None)), + ) + warnings.warn( + f"No default units for {coord_name} coord of {cube.name()}. " + "Meta-data may be incomplete." + ) new_coord = iris.coords.AuxCoord( np.array(coord_val, dtype=np.float32), bounds=bounds, **coord_keys ) @@ -830,7 +812,7 @@ def probability_coord(cube, field): if field.threshold_type == 3: pass else: - if is_multi_member_field: + if is_probability_field: cube.units = "1" cube.rename(f"probability_of_{cube.name()}") @@ -842,15 +824,27 @@ def probability_coord(cube, field): probability_attributes.append(probmethod_lookup[key]) num = num - key cube.attributes["Probability methods"] = probability_attributes - if field.member_count == 1: - is_multi_member_field = False - return is_multi_member_field + return def soil_type_coord(cube, field): """Add soil type as a coord if appropriate""" + soil_type_codes = { + 1: "broadleaf_tree", + 2: "needleleaf_tree", + 3: "c3_grass", + 4: "c4_grass", + 5: "crop", + 6: "shrub", + 7: "urban", + 8: "water", + 9: "soil", + 10: "ice", + 601: "urban_canyon", + 602: "urban_roof", + } if field.threshold_type != 0: - soil_name = SOIL_TYPE_CODES.get(field.soil_type, None) + soil_name = soil_type_codes.get(field.soil_type, None) if soil_name: cube.add_aux_coord( iris.coords.AuxCoord( @@ -861,11 +855,28 @@ def soil_type_coord(cube, field): def time_averaging(cube, field): """Decode the averagingtype code - similar to the PP LBPROC code.""" + time_averaging_codes = { + 8192: "maximum in period", + 4096: "minimum in period", + 2048: "unknown(2048)", + 1024: "unknown(1024)", + 512: "time lagged", + 256: "extrapolation", + 128: "accumulation or average", + 64: "from UM 150m", + 32: "scaled to UM resolution", + 16: "averaged over multiple surface types", + 8: "only observations used", + 4: "smoothed", + 2: "cold bias applied", + 1: "warm bias applied", + } + num = field.averagingtype averaging_attributes = [] - for key in sorted(TIME_AVERAGING_CODES.keys(), reverse=True): + for key in sorted(time_averaging_codes.keys(), reverse=True): if num >= key: - averaging_attributes.append(TIME_AVERAGING_CODES[key]) + averaging_attributes.append(time_averaging_codes[key]) num = num - key if averaging_attributes: cube.attributes["processing"] = averaging_attributes @@ -905,8 +916,8 @@ def run(field): # add other stuff, if present soil_type_coord(cube, field) - if not probability_coord(cube, field): - ensemble_member(cube, field) + probability_coord(cube, field) + ensemble_member(cube, field) time_averaging(cube, field) attributes(cube, field) diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml index 223c3e88a9..17970e1480 100644 --- a/lib/iris/tests/results/nimrod/probability_fields.cml +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -1123,7 +1123,7 @@ - + diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 702f4e06ce..266e24273f 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -91,6 +91,7 @@ def test_orography(self): field.vertical_coord_type = 1 field.vertical_coord = 8888 field.ensemble_member = field.int_mdi + field.threshold_value = field.int_mdi field.title = "(MOCK) 2km mean orography" field.units = "metres" field.source = "GLOBE DTM" From f52edb0c09e9d428b2629b688767da315b9e1643 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Mon, 18 May 2020 15:02:09 +0100 Subject: [PATCH 27/46] Ran black (I forgot - boo hoo) --- lib/iris/fileformats/nimrod_load_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 203e5512c8..4a1e0d979f 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -460,7 +460,7 @@ def horizontal_grid(cube, field): ) cube.add_dim_coord(x_coord, 1) points = np.linspace( - field.y_origin - (field.num_rows-1) * field.row_step, + field.y_origin - (field.num_rows - 1) * field.row_step, field.y_origin, field.num_rows, endpoint=True, From 6d034397308f24d361d32a75f0d079d07fa2393f Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 21 May 2020 14:14:59 +0100 Subject: [PATCH 28/46] Responses to second review - Improves doc-strings - Removes unused Nimrod meta-data entries - Improves variable name - Removes duplicated and unreachable errors --- lib/iris/fileformats/nimrod.py | 40 +++----------- lib/iris/fileformats/nimrod_load_rules.py | 64 +++++++---------------- 2 files changed, 25 insertions(+), 79 deletions(-) diff --git a/lib/iris/fileformats/nimrod.py b/lib/iris/fileformats/nimrod.py index 7d203030c5..e52450ed9a 100644 --- a/lib/iris/fileformats/nimrod.py +++ b/lib/iris/fileformats/nimrod.py @@ -47,7 +47,7 @@ "num_model_levels", "proj_biaxial_ellipsoid", "ensemble_member", - "spare1", + "model_origin_id", "averagingtype", ) @@ -80,7 +80,7 @@ "tl_y", "tl_x", "tr_y", - "ty_x", + "tr_x", "br_y", "br_x", "bl_y", @@ -94,34 +94,6 @@ "recursive_filter_alpha", "threshold_fuzziness", "threshold_duration_fuzziness", - "data_header_float32_05", - "data_header_float32_06", - "data_header_float32_07", - "data_header_float32_08", - "data_header_float32_09", - "data_header_float32_10", - "data_header_float32_11", - "data_header_float32_12", - "data_header_float32_13", - "data_header_float32_14", - "data_header_float32_15", - "data_header_float32_16", - "data_header_float32_17", - "data_header_float32_18", - "data_header_float32_19", - "data_header_float32_20", - "data_header_float32_21", - "data_header_float32_22", - "data_header_float32_23", - "data_header_float32_24", - "data_header_float32_25", - "data_header_float32_26", - "data_header_float32_27", - "data_header_float32_28", - "data_header_float32_29", - "data_header_float32_30", - "data_header_float32_31", - "data_header_float32_32", ) @@ -195,6 +167,11 @@ class NimrodField: Capable of converting itself into a :class:`~iris.cube.Cube` + References: + Met Office (2003): Met Office Rain Radar Data from the NIMROD System. + NCAS British Atmospheric Data Centre, date of citation. + http://catalogue.ceda.ac.uk/uuid/82adec1f896af6169112d09cc1174499 + """ def __init__(self, from_file=None): @@ -298,9 +275,6 @@ def _read_data(self, infile): "Expected data leading_length of %d" % num_data_bytes ) - # TODO: Deal appropriately with MDI. Can't just create masked arrays - # as cube merge converts masked arrays with no masks to ndarrays, - # thus mergable cube can split one mergable cube into two. self.data = np.fromfile(infile, dtype=numpy_dtype, count=num_data) if sys.byteorder == "little": diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 4a1e0d979f..5fd42fa73a 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -22,9 +22,6 @@ __all__ = ["run"] -# Meridian scaling for British National grid. -MERIDIAN_SCALING_BNG = 0.9996012717 - NIMROD_DEFAULT = -32767.0 TIME_UNIT = cf_units.Unit( @@ -199,7 +196,7 @@ def units(cube, field): def time(cube, field): - """Add a time coord to the cube.""" + """Add a time coord to the cube based on validity time and time-window.""" if field.vt_year <= 0: # Some ancillary files, eg land sea mask do not # have a validity time. @@ -215,16 +212,16 @@ def time(cube, field): ) point = np.around(TIME_UNIT.date2num(valid_date)).astype(np.int64) - lb_delta = None + period_seconds = None if field.period_minutes == 32767: - lb_delta = field.period_seconds + period_seconds = field.period_seconds elif ( not is_missing(field, field.period_minutes) and field.period_minutes != 0 ): - lb_delta = field.period_minutes * 60 - if lb_delta: - bounds = np.array([point - lb_delta, point], dtype=np.int64) + period_seconds = field.period_minutes * 60 + if period_seconds: + bounds = np.array([point - period_seconds, point], dtype=np.int64) else: bounds = None @@ -265,17 +262,9 @@ def forecast_period(cube): frt_coord = cube.coord("forecast_reference_time") except CoordinateNotFoundError: return - if len(time_coord.points) != 1 or len(frt_coord.points) != 1: - raise TranslationError( - "Unexpected number of points on time coordinates. Expected time:1; " - f"forecast_reference_time:1. Got {len(time_coord.points)}; " - f"{len(frt_coord.points)}" - ) time_delta = time_coord.cell(0).point - frt_coord.cell(0).point - points = np.array( - time_delta.days * 24 * 60 * 60 + time_delta.seconds, dtype=np.int32 - ) + points = np.array(time_delta.total_seconds(), dtype=np.int32) forecast_period_unit = cf_units.Unit("second") if cube.coord("time").has_bounds(): time_window = time_coord.cell(0).bound @@ -430,10 +419,6 @@ def horizontal_grid(cube, field): """Add X and Y coordinates to the cube. """ - if field.origin_corner != 0: - raise TranslationError( - "Corner {0} not yet implemented".format(field.origin_corner) - ) crs = coord_system(field) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: units_name = "m" @@ -473,7 +458,8 @@ def horizontal_grid(cube, field): def vertical_coord(cube, field): - """Add a vertical coord to the cube, if appropriate.""" + """Add a vertical coord to the cube, with bounds, if appropriate. + Handles special numbers for "at-sea-level" (8888) and "at-ground-level" (9999).""" # vertical_codes contains conversions from the Nimrod Documentation for the # header entry 20 for the vertical coordinate type # Unhandled vertical_codes values (no use case identified): @@ -629,20 +615,6 @@ def add_attr(item): add_attr("sat_space_count") add_attr("ducting_index") add_attr("elevation_angle") - add_attr("radar_num") - add_attr("radars_bitmask") - add_attr("more_radars_bitmask") - add_attr("clutter_map_num") - add_attr("calibration_type") - add_attr("bright_band_height") - add_attr("bright_band_intensity") - add_attr("bright_band_test1") - add_attr("bright_band_test2") - add_attr("infill_flag") - add_attr("stop_elevation") - add_attr("sensor_id") - add_attr("meteosat_id") - add_attr("alphas_available") cube_source = field.source.strip() # Handle a few known meta-data errors: @@ -652,15 +624,14 @@ def add_attr(item): cube_source = "Nimrod pwind routine" if field.source.strip() == "pwind": cube_source = "Nimrod pwind routine" - if "radar" not in cube_source: - for key in [ - "neighbourhood_radius", - "recursive_filter_iterations", - "recursive_filter_alpha", - "threshold_vicinity_radius", - "probability_period_of_event", - ]: - add_attr(key) + for key in [ + "neighbourhood_radius", + "recursive_filter_iterations", + "recursive_filter_alpha", + "threshold_vicinity_radius", + "probability_period_of_event", + ]: + add_attr(key) rematcher = re.compile(r"^ek\d\d$") if ( @@ -676,6 +647,7 @@ def add_attr(item): def known_threshold_coord(field): """ Supplies known threshold coord meta-data for known use cases. + threshold_value_alt exists because some meta-data are mis-assigned in the Nimrod data. """ coord_keys = {} if field.field_code == 161 and field.threshold_value >= 0.0: From 0913e4b4827b895e9db09c065b09508ce32065ec Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 13:36:12 +0100 Subject: [PATCH 29/46] Simplifies dtype setting - This changes a number of results that were being promoted to float32 when int16 was appropriate. - Fixes a meta-data issue with certain percentile data. - Adds some more comments around building probability coordinates. --- lib/iris/fileformats/nimrod_load_rules.py | 56 +++++++++++++------ .../results/nimrod/probability_fields.cml | 50 ++++++++--------- .../nimrod/u1096_ng_ek00_cloud_2km.cml | 24 ++++---- .../nimrod/u1096_ng_ek00_convection_2km.cml | 20 +++---- .../nimrod/u1096_ng_ek00_convwind_2km.cml | 12 ++-- .../nimrod/u1096_ng_ek00_frzlev_2km.cml | 32 +++++------ .../nimrod/u1096_ng_ek00_height_2km.cml | 4 +- .../nimrod/u1096_ng_ek00_preciptype_2km.cml | 12 ++-- .../nimrod/u1096_ng_ek00_radiation_2km.cml | 4 +- .../results/nimrod/u1096_ng_ek00_wind_2km.cml | 4 +- .../results/nimrod/u1096_ng_ek01_cape_2km.cml | 24 ++++---- 11 files changed, 133 insertions(+), 109 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 5fd42fa73a..6df50a5f2c 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -140,7 +140,7 @@ def units(cube, field): field_units = remove_unprintable_chars(field.units) if field_units == "m/2-25k": # Handle strange visibility units - cube.data = (cube.data + 25000.0) * 2 + cube.data = (cube.data.astype(np.float32) + 25000.0) * 2 field_units = "m" if "*" in field_units: # Split into unit string and integer @@ -148,28 +148,28 @@ def units(cube, field): if "^" in unit_list[1]: # Split out magnitude unit_sublist = unit_list[1].split("^") - cube.data = cube.data / float(unit_sublist[0]) ** float( - unit_sublist[1] - ) + cube.data = cube.data.astype(np.float32) / float( + unit_sublist[0] + ) ** float(unit_sublist[1]) else: - cube.data = cube.data / float(unit_list[1]) + cube.data = cube.data.astype(np.float32) / float(unit_list[1]) field_units = unit_list[0] if "ug/m3E1" in field_units: # Split into unit string and integer unit_list = field_units.split("E") - cube.data = cube.data / 10.0 ** float(unit_list[1]) + cube.data = cube.data.astype(np.float32) / 10.0 ** float(unit_list[1]) field_units = unit_list[0] if field_units == "%": # Convert any percentages into fraction field_units = "1" - cube.data = cube.data / 100.0 + cube.data = cube.data.astype(np.float32) / 100.0 if field_units == "oktas": field_units = "1" - cube.data /= 8.0 + cube.data = cube.data.astype(np.float32) / 8.0 if field_units == "dBZ": # cf_units doesn't recognise decibels (dBZ), but does know BZ field_units = "BZ" - cube.data /= 10.0 + cube.data = cube.data.astype(np.float32) / 10.0 if not field_units: if field.field_code == 8: # Relative Humidity data are unitless, but not "unknown" @@ -194,6 +194,10 @@ def units(cube, field): ) cube.attributes["invalid_units"] = field_units + if cube.dtype == np.float64: + # Demote any float64 that may have arisen from unit conversions to float32. + cube.data = cube.data.astype(np.float32) + def time(cube, field): """Add a time coord to the cube based on validity time and time-window.""" @@ -289,6 +293,7 @@ def mask_cube(cube, field): Updates cube.data to be a masked array if appropriate. """ + dtype = cube.dtype masked_points = None if field.datum_type == 1: # field.data are integers @@ -297,7 +302,9 @@ def mask_cube(cube, field): # field.data are floats masked_points = np.isclose(field.data, field.float32_mdi) if np.any(masked_points): - cube.data = np.ma.masked_array(cube.data, mask=masked_points) + cube.data = np.ma.masked_array( + cube.data, mask=masked_points, dtype=dtype + ) def experiment(cube, field): @@ -650,7 +657,11 @@ def known_threshold_coord(field): threshold_value_alt exists because some meta-data are mis-assigned in the Nimrod data. """ coord_keys = {} - if field.field_code == 161 and field.threshold_value >= 0.0: + if ( + field.field_code == 161 + and field.threshold_value >= 0.0 + and "pc" not in field.title + ): coord_keys = {"var_name": "threshold"} if field.threshold_value_alt > 8.0: coord_keys["standard_name"] = "height" @@ -733,7 +744,7 @@ def probability_coord(cube, field): } is_probability_field = False coord_keys = probtype_lookup.get(field.threshold_type, {}) - if coord_keys: + if coord_keys.get("var_name") == "threshold": is_probability_field = True coord_keys.update(known_threshold_coord(field)) if not coord_keys.get("units"): @@ -741,11 +752,19 @@ def probability_coord(cube, field): field.field_code, "unknown" ) coord_val = None + # coord_val could come from the threshold_value or threshold_value_alt: if field.threshold_value_alt > -32766.0: coord_val = field.threshold_value_alt elif field.threshold_value > -32766.0: coord_val = field.threshold_value - if cube.name().find("pc") > 0: + + # coord_val could also be encoded in the cube name if we have a percentile + # (this overrides the threshold_value which may be unrelated in the case of + # the 50th %ile of 3okta cloud cover) + if ( + coord_keys.get("long_name") == "percentile" + and cube.name().find("pc") > 0 + ): try: coord_val = [ int(x.strip("pc")) @@ -754,6 +773,8 @@ def probability_coord(cube, field): ][0] except IndexError: pass + + # If we found a coord_val, build the coord (with bounds) and add to cube) if coord_val is not None: if field.threshold_fuzziness > -32766.0: bounds = [ @@ -785,6 +806,11 @@ def probability_coord(cube, field): pass else: if is_probability_field: + # Some probability fields have inappropriate units (those of the threshold) + if "%" in cube.name(): + # If the cube name has % in it, convert to fraction + cube.rename(cube.name().replace("%", "fraction")) + cube.data = cube.data.astype(np.float32) / 100.0 cube.units = "1" cube.rename(f"probability_of_{cube.name()}") @@ -867,7 +893,7 @@ def run(field): * A new :class:`~iris.cube.Cube`, created from the NimrodField. """ - cube = iris.cube.Cube(field.data.astype(np.float32)) + cube = iris.cube.Cube(field.data) name(cube, field) mask_cube(cube, field) @@ -895,6 +921,4 @@ def run(field): origin_corner(cube, field) - cube.data = cube.data.astype(np.float32) - return cube diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml index 17970e1480..7b88107572 100644 --- a/lib/iris/tests/results/nimrod/probability_fields.cml +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -297,7 +297,7 @@ - + @@ -308,9 +308,6 @@ - - - @@ -320,6 +317,9 @@ + + + @@ -335,9 +335,9 @@ - + - + @@ -382,9 +382,9 @@ - + - + @@ -426,9 +426,9 @@ - + - + @@ -465,9 +465,9 @@ - + - + @@ -501,9 +501,9 @@ - + - + @@ -541,9 +541,9 @@ - + - + @@ -577,9 +577,9 @@ - + - + @@ -624,9 +624,9 @@ - + - + @@ -668,7 +668,7 @@ - + @@ -1089,7 +1089,7 @@ - + @@ -1134,7 +1134,7 @@ - + @@ -1309,7 +1309,7 @@ - + @@ -1350,7 +1350,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml index 5a43bc8596..c84f88e062 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml @@ -1,6 +1,6 @@ - + @@ -37,9 +37,9 @@ - + - + @@ -76,9 +76,9 @@ - + - + @@ -116,9 +116,9 @@ - + - + @@ -156,7 +156,7 @@ - + @@ -209,7 +209,7 @@ - + @@ -246,9 +246,9 @@ - + - + @@ -288,6 +288,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml index a98f3d4147..6ece3386a4 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml @@ -1,6 +1,6 @@ - + @@ -44,9 +44,9 @@ - + - + @@ -83,9 +83,9 @@ - + - + @@ -122,7 +122,7 @@ - + @@ -214,7 +214,7 @@ - + @@ -251,9 +251,9 @@ - + - + @@ -290,7 +290,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml index 610dbd951f..2f52a93277 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convwind_2km.cml @@ -1,6 +1,6 @@ - + @@ -44,9 +44,9 @@ - + - + @@ -90,7 +90,7 @@ - + @@ -184,7 +184,7 @@ - + @@ -228,7 +228,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml index f039e12d8b..09f6860a13 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml @@ -1,6 +1,6 @@ - + @@ -37,9 +37,9 @@ - + - + @@ -76,9 +76,9 @@ - + - + @@ -115,9 +115,9 @@ - + - + @@ -155,9 +155,9 @@ - + - + @@ -195,9 +195,9 @@ - + - + @@ -234,9 +234,9 @@ - + - + @@ -274,9 +274,9 @@ - + - + @@ -314,6 +314,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml index e2dc992c15..a01d45ef00 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml @@ -1,6 +1,6 @@ - + @@ -37,6 +37,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml index 701116d0c7..3fdf646e70 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_preciptype_2km.cml @@ -238,7 +238,7 @@ - + @@ -275,9 +275,9 @@ - + - + @@ -315,9 +315,9 @@ - + - + @@ -355,6 +355,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml index eebff3179d..b2e611e7d7 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml @@ -117,7 +117,7 @@ - + @@ -154,7 +154,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml index 6e6b2c0b69..b882df23ee 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml @@ -93,7 +93,7 @@ - + @@ -137,7 +137,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml index bfb716a8fc..1756ac0205 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek01_cape_2km.cml @@ -1,6 +1,6 @@ - + @@ -37,9 +37,9 @@ - + - + @@ -76,9 +76,9 @@ - + - + @@ -118,9 +118,9 @@ - + - + @@ -157,9 +157,9 @@ - + - + @@ -196,9 +196,9 @@ - + - + @@ -235,6 +235,6 @@ - + From 35ef71834d0c6a86564c7a7751415007b9ff3ea0 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 13:36:48 +0100 Subject: [PATCH 30/46] Improves matching of mean and spread in title to be case insensitive --- lib/iris/fileformats/nimrod_load_rules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 6df50a5f2c..8d9128253e 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -87,10 +87,10 @@ def name(cube, field): ) if field.ensemble_member == -98: - if "mean" not in cube_title: + if not re.match("(?i)^.*(mean).*", cube_title): cube_title = "mean_of_" + cube_title if field.ensemble_member == -99: - if "spread" not in cube_title: + if not re.match("(?i)^.*(spread).*", cube_title): cube_title = "standard_deviation_of_" + cube_title cube.rename(remove_unprintable_chars(cube_title)) From dcdfcd76003e66447411a6d45296a127d56e2afd Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 14:08:28 +0100 Subject: [PATCH 31/46] Removes bespoke MOGREPS-UK identifier but continues to remove realization number from cube source. --- lib/iris/fileformats/nimrod_load_rules.py | 18 +++++++---- .../results/nimrod/probability_fields.cml | 30 +++++++++---------- .../nimrod/u1096_ng_ek00_cloud3d0060_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_cloud_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_convection_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_frzlev_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_height_2km.cml | 2 +- .../nimrod/u1096_ng_ek00_precip_2km.cml | 6 ++-- .../nimrod/u1096_ng_ek00_precipaccum_2km.cml | 2 +- .../nimrod/u1096_ng_ek00_pressure_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_radiation_2km.cml | 14 ++++----- .../nimrod/u1096_ng_ek00_radiationuv_2km.cml | 6 ++-- .../results/nimrod/u1096_ng_ek00_refl_2km.cml | 2 +- .../u1096_ng_ek00_relhumidity3d0060_2km.cml | 2 +- .../results/nimrod/u1096_ng_ek00_snow_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_soil3d0060_2km.cml | 8 ++--- .../results/nimrod/u1096_ng_ek00_soil_2km.cml | 6 ++-- .../nimrod/u1096_ng_ek00_temperature_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_visibility_2km.cml | 6 ++-- .../results/nimrod/u1096_ng_ek00_wind_2km.cml | 4 +-- .../nimrod/u1096_ng_ek00_winduv3d0015_2km.cml | 4 +-- ...u1096_ng_ek07_precip0540_accum180_18km.cml | 2 +- 22 files changed, 73 insertions(+), 67 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 8d9128253e..b782b2bd18 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -640,12 +640,18 @@ def add_attr(item): ]: add_attr(key) - rematcher = re.compile(r"^ek\d\d$") - if ( - rematcher.match(cube_source) is not None - or cube_source.find("umek") == 0 - ): - cube_source = "MOGREPS-UK" + # Remove member number from cube_source + match = re.match( + r"^(?P\w\w)(?P\d\d)$", cube_source + ) + try: + r_coord = cube.coord("realization") + except CoordinateNotFoundError: + r_coord = None + if match is not None: + if r_coord: + if int(match["realization"]) == r_coord.points[0]: + cube_source = match["model_code"] cube.attributes["source"] = cube_source cube.attributes["title"] = "Unknown" cube.attributes["institution"] = "Met Office" diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml index 7b88107572..2a046db2a0 100644 --- a/lib/iris/tests/results/nimrod/probability_fields.cml +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -677,7 +677,7 @@ - + @@ -715,7 +715,7 @@ - + @@ -755,7 +755,7 @@ - + @@ -796,7 +796,7 @@ - + @@ -842,7 +842,7 @@ - + @@ -890,7 +890,7 @@ - + @@ -938,7 +938,7 @@ - + @@ -981,7 +981,7 @@ - + @@ -1142,7 +1142,7 @@ - + @@ -1358,7 +1358,7 @@ - + @@ -1405,7 +1405,7 @@ - + @@ -1457,7 +1457,7 @@ - + @@ -1746,7 +1746,7 @@ - + @@ -1789,7 +1789,7 @@ - + @@ -1833,7 +1833,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml index 5ef167f1ac..68ec95555c 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud3d0060_2km.cml @@ -6,7 +6,7 @@ - + @@ -62,7 +62,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml index c84f88e062..7253f93902 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml @@ -6,7 +6,7 @@ - + @@ -45,7 +45,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml index 6ece3386a4..e6c99f9e50 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_convection_2km.cml @@ -298,7 +298,7 @@ - + @@ -340,7 +340,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml index 09f6860a13..b2b47715a2 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_frzlev_2km.cml @@ -6,7 +6,7 @@ - + @@ -45,7 +45,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml index a01d45ef00..4fb1371250 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_height_2km.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml index 8260ce8ee6..a57361795d 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml @@ -6,7 +6,7 @@ - + @@ -46,7 +46,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml index 8268297e22..25d10ec94d 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml @@ -7,7 +7,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml index 9d354ec469..edb0862676 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_pressure_2km.cml @@ -6,7 +6,7 @@ - + @@ -45,7 +45,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml index b2e611e7d7..38f076f232 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiation_2km.cml @@ -6,7 +6,7 @@ - + @@ -45,7 +45,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -123,7 +123,7 @@ - + @@ -162,7 +162,7 @@ - + @@ -201,7 +201,7 @@ - + @@ -240,7 +240,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml index de10bd934c..35bed38591 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_radiationuv_2km.cml @@ -6,7 +6,7 @@ - + @@ -45,7 +45,7 @@ - + @@ -84,7 +84,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml index 7d8bd30fdb..4411ff9dd5 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_refl_2km.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml index bf2f96a5f6..8759dac5c7 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_relhumidity3d0060_2km.cml @@ -6,7 +6,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml index b2bfe10e9c..ce549ab3cd 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_snow_2km.cml @@ -6,7 +6,7 @@ - + @@ -46,7 +46,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml index 0c257197d6..9385bfc9ae 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil3d0060_2km.cml @@ -6,7 +6,7 @@ - + @@ -52,7 +52,7 @@ - + @@ -98,7 +98,7 @@ - + @@ -144,7 +144,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml index 1bdda8f9cf..a76971a1ed 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_soil_2km.cml @@ -6,7 +6,7 @@ - + @@ -45,7 +45,7 @@ - + @@ -89,7 +89,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml index 354e4d7be0..09677ff57a 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_temperature_2km.cml @@ -7,7 +7,7 @@ - + @@ -54,7 +54,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml index f2a9e7cfa4..8a0f50700c 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_visibility_2km.cml @@ -6,7 +6,7 @@ - + @@ -52,7 +52,7 @@ - + @@ -98,7 +98,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml index b882df23ee..df2054e8af 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_wind_2km.cml @@ -191,7 +191,7 @@ - + @@ -238,7 +238,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml index b3177568ae..331ff59c74 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_winduv3d0015_2km.cml @@ -6,7 +6,7 @@ - + @@ -52,7 +52,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml index 7ad5bda993..1641f8779e 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml @@ -7,7 +7,7 @@ - + From 2e3079fc3b885387579fc77df3ab402a8abb9967 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 14:09:06 +0100 Subject: [PATCH 32/46] Adds references for ellipsoids --- lib/iris/fileformats/nimrod_load_rules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index b782b2bd18..fe2a68156c 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -324,10 +324,11 @@ def proj_biaxial_ellipsoid(field): missing, select the right pre-defined ellipsoid dictionary (Airy_1830 or international_1924). + References: + Airy 1830: https://georepository.com/ellipsoid_7001/Airy-1830.html + International 1924: https://georepository.com/ellipsoid_7022/International-1924.html + """ - # Reference for airy_1830 and international_1924 ellipsoids: - # http://fcm9/projects/PostProc/wiki/PostProcDocDomains#ProjectionConstants - # Reference for GRS80: airy_1830 = { "semi_major_axis": 6377563.396, "semi_minor_axis": 6356256.910, From ac1bf9ee4cf1658b1249d296bf773f9dfce9474f Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 14:38:25 +0100 Subject: [PATCH 33/46] Removes unnecessary test in soil_type_coord --- lib/iris/fileformats/nimrod_load_rules.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index fe2a68156c..59a6fe666d 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -848,14 +848,13 @@ def soil_type_coord(cube, field): 601: "urban_canyon", 602: "urban_roof", } - if field.threshold_type != 0: - soil_name = soil_type_codes.get(field.soil_type, None) - if soil_name: - cube.add_aux_coord( - iris.coords.AuxCoord( - soil_name, standard_name="soil_type", units=None - ) + soil_name = soil_type_codes.get(field.soil_type, None) + if soil_name: + cube.add_aux_coord( + iris.coords.AuxCoord( + soil_name, standard_name="soil_type", units=None ) + ) def time_averaging(cube, field): From f7aec6272c10db2fcc7bd7a8a6d3f7d421ba5951 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 14:48:01 +0100 Subject: [PATCH 34/46] Added element information to header lists --- lib/iris/fileformats/nimrod.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/iris/fileformats/nimrod.py b/lib/iris/fileformats/nimrod.py index e52450ed9a..2f4e32ed95 100644 --- a/lib/iris/fileformats/nimrod.py +++ b/lib/iris/fileformats/nimrod.py @@ -16,7 +16,7 @@ import iris.fileformats.nimrod_load_rules -# general header (int16) elements +# general header (int16) elements 1-31 (Fortran bytes 1-62) general_header_int16s = ( "vt_year", "vt_month", @@ -52,7 +52,7 @@ ) -# general header (float32) elements +# general header (float32) elements 32-59 (Fortran bytes 63-174) general_header_float32s = ( "vertical_coord", "reference_vertical_coord", @@ -75,7 +75,7 @@ ) -# data specific header (float32) elements +# data specific header (float32) elements 60-104 (Fortran bytes 175-354) data_header_float32s = ( "tl_y", "tl_x", @@ -97,7 +97,11 @@ ) -# data specific header (int16) elements +# data specific header (char) elements 105-107 (bytes 355-410) +# units, source and title + + +# data specific header (int16) elements 108-159 (Fortran bytes 411-512) data_header_int16s = ( "threshold_type", "probability_method", From 8d8feab1b8ff0f433e15de8391bfdc9339dd1546 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 14:53:54 +0100 Subject: [PATCH 35/46] Second review: - Distinguishes between precipitation rate and accumulation --- lib/iris/fileformats/nimrod_load_rules.py | 4 ++-- .../tests/results/nimrod/probability_fields.cml | 16 ++++++++-------- .../results/nimrod/u1096_ng_ek00_precip_2km.cml | 2 +- .../nimrod/u1096_ng_ek00_precipaccum_2km.cml | 2 +- .../u1096_ng_ek07_precip0540_accum180_18km.cml | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 59a6fe666d..861259830c 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -50,8 +50,8 @@ def name(cube, field): 27: "snow fraction", 28: "snow probability", 58: "temperature", - 61: "precipitation", - 63: "precipitation", + 61: "amount_of_precipitation", + 63: "rate_of_precipitation", 101: "snow_melting_level_above_sea_level", 102: "rain_melted_level_above_sea_level", 155: "Visibility", diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml index 2a046db2a0..eb54c785c6 100644 --- a/lib/iris/tests/results/nimrod/probability_fields.cml +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -670,7 +670,7 @@ - + @@ -707,7 +707,7 @@ - + @@ -748,7 +748,7 @@ - + @@ -785,7 +785,7 @@ - + @@ -834,7 +834,7 @@ - + @@ -879,7 +879,7 @@ - + @@ -928,7 +928,7 @@ - + @@ -975,7 +975,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml index a57361795d..59776b5b74 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precip_2km.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml index 25d10ec94d..0fa98e3bb6 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_precipaccum_2km.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml index 1641f8779e..f4710dd36d 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek07_precip0540_accum180_18km.cml @@ -1,6 +1,6 @@ - + From bf68e4c04d534bb0f4704a58119621b32209abe6 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 15:06:17 +0100 Subject: [PATCH 36/46] Second review: - Raises a warning if the coordinate reference system is incomplete --- lib/iris/fileformats/nimrod_load_rules.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 861259830c..3fbac60aa3 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -260,6 +260,9 @@ def forecast_period(cube): """ Add a forecast_period coord based on existing time and forecast_reference_time coords. + + Must be run after time() and reference_time() + """ try: time_coord = cube.coord("time") @@ -404,13 +407,20 @@ def coord_system(field): # Some Radarnet files are missing these. set_british_national_grid_defaults(field) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: - coord_sys = iris.coord_systems.TransverseMercator( + crs_args = ( field.true_origin_latitude, field.true_origin_longitude, field.true_origin_easting, field.true_origin_northing, field.tm_meridian_scaling, - iris.coord_systems.GeogCS(**ellipsoid), + ) + if any([is_missing(field, v) for v in crs_args]): + warnings.warn( + f"Coordinate Reference System is not completely defined. " + f"Plotting and reprojection may be impaired." + ) + coord_sys = iris.coord_systems.TransverseMercator( + *crs_args, iris.coord_systems.GeogCS(**ellipsoid), ) elif field.horizontal_grid_type == 1: coord_sys = iris.coord_systems.GeogCS(**ellipsoid) From 60ed07e0b36f15f798c6de08e0f887e7e6f20d70 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 15:12:06 +0100 Subject: [PATCH 37/46] Second review: - Expands doc-strings for coord system functions --- lib/iris/fileformats/nimrod_load_rules.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 3fbac60aa3..07acc19c49 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -399,7 +399,12 @@ def set_british_national_grid_defaults(field): def coord_system(field): - """Define the coordinate system for the field.""" + """Define the coordinate system for the field. + Handles Transverse Mercator, Universal Transverse Mercator and Plate Carree. + + Transverse Mercator projections will default to the British National Grid if any + parameters are missing. + """ ellipsoid = proj_biaxial_ellipsoid(field) if field.horizontal_grid_type == 0: @@ -425,16 +430,15 @@ def coord_system(field): elif field.horizontal_grid_type == 1: coord_sys = iris.coord_systems.GeogCS(**ellipsoid) else: - raise TranslationError( - "Coordinate system for field type {} not implemented".format( - field.horizontal_grid_type - ) - ) + coord_sys = None return coord_sys def horizontal_grid(cube, field): """Add X and Y coordinates to the cube. + Handles Transverse Mercator, Universal Transverse Mercator and Plate Carree. + + coordinate reference system is supplied by coord_system(field) """ crs = coord_system(field) From f95a8f1ac62639b91bb544ddce34d2481ab2ea88 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 15:16:53 +0100 Subject: [PATCH 38/46] Second review: - Rearranges warning message to make it easier to read. --- lib/iris/fileformats/nimrod_load_rules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 07acc19c49..942ebf51d1 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -536,9 +536,8 @@ def vertical_coord(cube, field): ): msg = ( "Unmatched vertical coord types " - f"{field.vertical_coord_type} != " - f"{field.reference_vertical_coord_type}" - f". Assuming {field.vertical_coord_type}" + f"{field.vertical_coord_type} != {field.reference_vertical_coord_type}. " + f"Assuming {field.vertical_coord_type}" ) warnings.warn(msg) From f2a5dc630658b34899a8d0bce2c9260abc5a44d9 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Fri, 22 May 2020 15:25:35 +0100 Subject: [PATCH 39/46] Second review: - Moves origin_corner() to before horizontal_grid() --- lib/iris/fileformats/nimrod_load_rules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 942ebf51d1..3b87d2d041 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -926,6 +926,7 @@ def run(field): experiment(cube, field) # horizontal grid + origin_corner(cube, field) horizontal_grid(cube, field) # vertical @@ -938,6 +939,4 @@ def run(field): time_averaging(cube, field) attributes(cube, field) - origin_corner(cube, field) - return cube From 66079d863dc4a89bbf2ab5c9113b2f5f792f06a2 Mon Sep 17 00:00:00 2001 From: "fiona.rust" Date: Tue, 26 May 2020 15:26:21 +0100 Subject: [PATCH 40/46] Adding unit tests for unit function --- lib/iris/fileformats/nimrod_load_rules.py | 10 +-- .../nimrod_load_rules/test_units.py | 88 ++++++++++++++++++- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 3b87d2d041..2715209760 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -128,7 +128,6 @@ def units(cube, field): "Code": "1", "mask": "1", "mb": "hPa", - "g/Kg": "1", "unitless": "1", "Fraction": "1", "index": "1", @@ -157,7 +156,7 @@ def units(cube, field): if "ug/m3E1" in field_units: # Split into unit string and integer unit_list = field_units.split("E") - cube.data = cube.data.astype(np.float32) / 10.0 ** float(unit_list[1]) + cube.data = cube.data.astype(np.float32) / 10.0 field_units = unit_list[0] if field_units == "%": # Convert any percentages into fraction @@ -170,6 +169,9 @@ def units(cube, field): # cf_units doesn't recognise decibels (dBZ), but does know BZ field_units = "BZ" cube.data = cube.data.astype(np.float32) / 10.0 + if field_units == "g/Kg": + field_units = "kg/kg" + cube.data = cube.data.astype(np.float32) / 1000.0 if not field_units: if field.field_code == 8: # Relative Humidity data are unitless, but not "unknown" @@ -194,10 +196,6 @@ def units(cube, field): ) cube.attributes["invalid_units"] = field_units - if cube.dtype == np.float64: - # Demote any float64 that may have arisen from unit conversions to float32. - cube.data = cube.data.astype(np.float32) - def time(cube, field): """Add a time coord to the cube based on validity time and time-window.""" diff --git a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py index 019f523a1c..cf6dd8cdb3 100644 --- a/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py +++ b/lib/iris/tests/unit/fileformats/nimrod_load_rules/test_units.py @@ -33,9 +33,7 @@ def setUp(self): float32_mdi=NIMROD_DEFAULT, spec=NimrodField, ) - self.cube = mock.Mock( - data=np.zeros((3, 3), dtype=np.float32), spec=Cube - ) + self.cube = Cube(np.ones((3, 3), dtype=np.float32)) def _call_units( self, data=None, units_str=None, @@ -48,7 +46,7 @@ def _call_units( def test_null(self): with mock.patch("warnings.warn") as warn: - self._call_units(data=np.ones_like(self.cube.data), units_str="m") + self._call_units(units_str="m") self.assertEqual(warn.call_count, 0) self.assertEqual(self.cube.units, "m") self.assertArrayAlmostEqual( @@ -65,6 +63,88 @@ def test_times32(self): self.assertArrayAlmostEqual( self.cube.data, np.ones_like(self.cube.data) ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_visibility_units(self): + with mock.patch("warnings.warn") as warn: + self._call_units( + data=((np.ones_like(self.cube.data) / 2) - 25000), + units_str="m/2-25k", + ) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "m") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_power_in_units(self): + with mock.patch("warnings.warn") as warn: + self._call_units( + data=np.ones_like(self.cube.data) * 1000, units_str="mm*10^3" + ) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "mm") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_ug_per_m3_units(self): + with mock.patch("warnings.warn") as warn: + self._call_units( + data=((np.ones_like(self.cube.data) * 10)), + units_str="ug/m3E1", + ) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "ug/m3") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_g_per_kg(self): + with mock.patch("warnings.warn") as warn: + self._call_units( + data=((np.ones_like(self.cube.data) * 1000)), units_str="g/Kg", + ) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "kg/kg") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_unit_expection_dictionary(self): + with mock.patch("warnings.warn") as warn: + self._call_units(units_str="mb",) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "hPa") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_per_second(self): + with mock.patch("warnings.warn") as warn: + self._call_units(units_str="/s",) + self.assertEqual(warn.call_count, 0) + self.assertEqual(self.cube.units, "s^-1") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + + def test_unhandled_unit(self): + with mock.patch("warnings.warn") as warn: + self._call_units(units_str="kittens",) + self.assertEqual(warn.call_count, 1) + self.assertEqual(self.cube.units, "") + self.assertArrayAlmostEqual( + self.cube.data, np.ones_like(self.cube.data) + ) + self.assertEqual(self.cube.data.dtype, np.float32) + self.assertEqual(self.cube.attributes["invalid_units"], "kittens") if __name__ == "__main__": From 0178eeb812271d419b75079ebb515a9907dbddba Mon Sep 17 00:00:00 2001 From: "fiona.rust" Date: Tue, 26 May 2020 15:45:40 +0100 Subject: [PATCH 41/46] Removed unnecessary dictionaries from name function --- lib/iris/fileformats/nimrod_load_rules.py | 25 ++++++----------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 2715209760..79566e8ac3 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -44,6 +44,7 @@ def name(cube, field): """Set the cube's name from the field. Modifies the Nimrod object title based on other meta-data in the Nimrod field and known use cases. + Adds "mean_of" or "standard_deviation_of_" to the cube name if appropriate. """ title_from_field_code = { 12: "air_pressure", @@ -52,10 +53,12 @@ def name(cube, field): 58: "temperature", 61: "amount_of_precipitation", 63: "rate_of_precipitation", + 29: "fog fraction", 101: "snow_melting_level_above_sea_level", 102: "rain_melted_level_above_sea_level", 155: "Visibility", 156: "Worst visibility in grid point", + 161: "minimum_cloud_base_above_threshold", 172: "cloud_area_fraction_in_atmosphere", 218: "snowfall", 421: "precipitation type", @@ -63,29 +66,13 @@ def name(cube, field): 508: "low_level_jet_u_component", 509: "low_level_jet_curvature", 514: "low_level_jet_v_component", - 817: "wind_speed_of_gust", - 821: "Probabilistic Gust Risk Analysis from Observations", - } - # threshold_value >= 0 - title_from_field_code_and_threshold = { - 29: "fog fraction", - 161: "minimum_cloud_base_above_threshold", - } - # vertical_coord >= 0 - title_from_field_code_and_vcoord = { 804: "wind speed", 806: "wind direction", + 817: "wind_speed_of_gust", + 821: "Probabilistic Gust Risk Analysis from Observations", } - cube_title = title_from_field_code.get(field.field_code, field.title) - if field.threshold_value >= 0: - cube_title = title_from_field_code_and_threshold.get( - field.field_code, cube_title - ) - if field.vertical_coord >= 0: - cube_title = title_from_field_code_and_vcoord.get( - field.field_code, cube_title - ) + cube_title = title_from_field_code.get(field.field_code, field.title) if field.ensemble_member == -98: if not re.match("(?i)^.*(mean).*", cube_title): cube_title = "mean_of_" + cube_title From 454d57aecf85a8db23848cc92e5a892fabdcf4eb Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 28 May 2020 10:09:27 +0100 Subject: [PATCH 42/46] Improved comments for second review --- lib/iris/fileformats/nimrod_load_rules.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 3b87d2d041..ca556ea2b6 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -440,6 +440,7 @@ def horizontal_grid(cube, field): coordinate reference system is supplied by coord_system(field) + Must be run AFTER origin_corner() """ crs = coord_system(field) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: @@ -654,7 +655,9 @@ def add_attr(item): ]: add_attr(key) - # Remove member number from cube_source + # Remove member number from cube_source. This commonly takes the form ek04 where ek + # indicates the model and 04 is the realization number. As the number is represented + # by a realization coord, stripping it from here allows cubes to be merged. match = re.match( r"^(?P\w\w)(?P\d\d)$", cube_source ) From e6f70e0818679ec464a0f553155efb0982a7eb4f Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 28 May 2020 10:51:22 +0100 Subject: [PATCH 43/46] Adds kwarg to allow user to skip known meta-data inconsistencies. --- lib/iris/fileformats/nimrod.py | 9 ++- lib/iris/fileformats/nimrod_load_rules.py | 81 +++++++++++++---------- lib/iris/tests/test_nimrod.py | 2 +- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/lib/iris/fileformats/nimrod.py b/lib/iris/fileformats/nimrod.py index 2f4e32ed95..554b6e1a1d 100644 --- a/lib/iris/fileformats/nimrod.py +++ b/lib/iris/fileformats/nimrod.py @@ -294,7 +294,7 @@ def _read_data(self, infile): self.data = self.data.reshape(self.num_rows, self.num_cols) -def load_cubes(filenames, callback=None): +def load_cubes(filenames, callback=None, handle_metadata_errors=True): """ Loads cubes from a list of NIMROD filenames. @@ -307,6 +307,9 @@ def load_cubes(filenames, callback=None): * callback - a function which can be passed on to :func:`iris.io.run_callback` + * handle_metadata_errors - Set to False to omit handling of known meta-data deficiencies + in Nimrod-format data + .. note:: The resultant cubes may not be in the same order as in the files. @@ -325,7 +328,9 @@ def load_cubes(filenames, callback=None): # End of file. Move on to the next file. break - cube = iris.fileformats.nimrod_load_rules.run(field) + cube = iris.fileformats.nimrod_load_rules.run( + field, handle_metadata_errors=handle_metadata_errors + ) # Were we given a callback? if callback is not None: diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index 231ba8a7d9..cb737bddaa 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -40,7 +40,7 @@ def is_missing(field, value): ) -def name(cube, field): +def name(cube, field, handle_metadata_errors): """Set the cube's name from the field. Modifies the Nimrod object title based on other meta-data in the Nimrod field and known use cases. @@ -71,8 +71,10 @@ def name(cube, field): 817: "wind_speed_of_gust", 821: "Probabilistic Gust Risk Analysis from Observations", } - - cube_title = title_from_field_code.get(field.field_code, field.title) + if handle_metadata_errors: + cube_title = title_from_field_code.get(field.field_code, field.title) + else: + cube_title = field.title if field.ensemble_member == -98: if not re.match("(?i)^.*(mean).*", cube_title): cube_title = "mean_of_" + cube_title @@ -303,7 +305,7 @@ def experiment(cube, field): ) -def proj_biaxial_ellipsoid(field): +def proj_biaxial_ellipsoid(field, handle_metadata_errors): """ Returns the correct dictionary of arguments needed to define an iris.coord_systems.GeogCS. @@ -329,7 +331,10 @@ def proj_biaxial_ellipsoid(field): ellipsoid = airy_1830 elif field.proj_biaxial_ellipsoid == 1: ellipsoid = international_1924 - elif is_missing(field, field.proj_biaxial_ellipsoid): + elif ( + is_missing(field, field.proj_biaxial_ellipsoid) + and handle_metadata_errors + ): if field.horizontal_grid_type == 0: ellipsoid = airy_1830 elif ( @@ -351,29 +356,30 @@ def proj_biaxial_ellipsoid(field): return ellipsoid -def set_british_national_grid_defaults(field): +def set_british_national_grid_defaults(field, handle_metadata_errors): """Check for missing coord-system meta-data and set default values for the Ordnance Survey GB Transverse Mercator projection. Some Radarnet files are missing these.""" - if is_missing(field, field.true_origin_latitude): - field.true_origin_latitude = 49.0 - if is_missing(field, field.true_origin_longitude): - field.true_origin_longitude = -2.0 - if is_missing(field, field.true_origin_easting) or np.isclose( - # Some old files misquote the value in km instead of m - field.true_origin_easting, - 400.0, - ): - field.true_origin_easting = 400000.0 - if is_missing(field, field.true_origin_northing) or np.isclose( - # Some old files misquote the value in km instead of m - field.true_origin_northing, - -100.0, - ): - field.true_origin_northing = -100000.0 - if is_missing(field, field.tm_meridian_scaling): - field.tm_meridian_scaling = 0.9996012717 + if handle_metadata_errors: + if is_missing(field, field.true_origin_latitude): + field.true_origin_latitude = 49.0 + if is_missing(field, field.true_origin_longitude): + field.true_origin_longitude = -2.0 + if is_missing(field, field.true_origin_easting) or np.isclose( + # Some old files misquote the value in km instead of m + field.true_origin_easting, + 400.0, + ): + field.true_origin_easting = 400000.0 + if is_missing(field, field.true_origin_northing) or np.isclose( + # Some old files misquote the value in km instead of m + field.true_origin_northing, + -100.0, + ): + field.true_origin_northing = -100000.0 + if is_missing(field, field.tm_meridian_scaling): + field.tm_meridian_scaling = 0.9996012717 ng_central_meridian_sf_dp = 0.9996012717 if abs(field.tm_meridian_scaling - ng_central_meridian_sf_dp) < 1.0e-04: @@ -383,19 +389,19 @@ def set_british_national_grid_defaults(field): field.tm_meridian_scaling = ng_central_meridian_sf_dp -def coord_system(field): +def coord_system(field, handle_metadata_errors): """Define the coordinate system for the field. Handles Transverse Mercator, Universal Transverse Mercator and Plate Carree. Transverse Mercator projections will default to the British National Grid if any parameters are missing. """ - ellipsoid = proj_biaxial_ellipsoid(field) + ellipsoid = proj_biaxial_ellipsoid(field, handle_metadata_errors) if field.horizontal_grid_type == 0: # Check for missing grid meta-data and insert OSGB definitions. # Some Radarnet files are missing these. - set_british_national_grid_defaults(field) + set_british_national_grid_defaults(field, handle_metadata_errors) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: crs_args = ( field.true_origin_latitude, @@ -419,7 +425,7 @@ def coord_system(field): return coord_sys -def horizontal_grid(cube, field): +def horizontal_grid(cube, field, handle_metadata_errors): """Add X and Y coordinates to the cube. Handles Transverse Mercator, Universal Transverse Mercator and Plate Carree. @@ -427,7 +433,7 @@ def horizontal_grid(cube, field): Must be run AFTER origin_corner() """ - crs = coord_system(field) + crs = coord_system(field, handle_metadata_errors) if field.horizontal_grid_type == 0 or field.horizontal_grid_type == 4: units_name = "m" x_coord_name = "projection_x_coordinate" @@ -705,7 +711,7 @@ def known_threshold_coord(field): return coord_keys -def probability_coord(cube, field): +def probability_coord(cube, field, handle_metadata_errors): """ Adds a coord relating to probability meta-data from the header to the cube if appropriate. @@ -754,7 +760,8 @@ def probability_coord(cube, field): coord_keys = probtype_lookup.get(field.threshold_type, {}) if coord_keys.get("var_name") == "threshold": is_probability_field = True - coord_keys.update(known_threshold_coord(field)) + if handle_metadata_errors: + coord_keys.update(known_threshold_coord(field)) if not coord_keys.get("units"): coord_keys["units"] = units_from_field_code.get( field.field_code, "unknown" @@ -772,6 +779,7 @@ def probability_coord(cube, field): if ( coord_keys.get("long_name") == "percentile" and cube.name().find("pc") > 0 + and handle_metadata_errors ): try: coord_val = [ @@ -887,7 +895,7 @@ def time_averaging(cube, field): cube.attributes["processing"] = averaging_attributes -def run(field): +def run(field, handle_metadata_errors=True): """ Convert a NIMROD field to an Iris cube. @@ -895,6 +903,9 @@ def run(field): * field - a :class:`~iris.fileformats.nimrod.NimrodField` + * handle_metadata_errors - Set to False to omit handling of known meta-data deficiencies + in Nimrod-format data + Returns: * A new :class:`~iris.cube.Cube`, created from the NimrodField. @@ -902,7 +913,7 @@ def run(field): """ cube = iris.cube.Cube(field.data) - name(cube, field) + name(cube, field, handle_metadata_errors) mask_cube(cube, field) units(cube, field) @@ -915,14 +926,14 @@ def run(field): # horizontal grid origin_corner(cube, field) - horizontal_grid(cube, field) + horizontal_grid(cube, field, handle_metadata_errors) # vertical vertical_coord(cube, field) # add other stuff, if present soil_type_coord(cube, field) - probability_coord(cube, field) + probability_coord(cube, field, handle_metadata_errors) ensemble_member(cube, field) time_averaging(cube, field) attributes(cube, field) diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 266e24273f..221fc9eb9c 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -96,7 +96,7 @@ def test_orography(self): field.units = "metres" field.source = "GLOBE DTM" - nimrod_load_rules.name(cube, field) + nimrod_load_rules.name(cube, field, handle_metadata_errors=True) nimrod_load_rules.units(cube, field) nimrod_load_rules.reference_time(cube, field) nimrod_load_rules.vertical_coord(cube, field) From a1519b37d9f31c72c44ced3010d2c47529e3114b Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 28 May 2020 11:39:15 +0100 Subject: [PATCH 44/46] Adds test for kwarg to allow user to skip known meta-data inconsistencies. --- lib/iris/tests/test_nimrod.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 221fc9eb9c..60076e378b 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -12,6 +12,7 @@ import iris import iris.fileformats.nimrod_load_rules as nimrod_load_rules +from iris.exceptions import TranslationError def mock_nimrod_field(): @@ -76,6 +77,20 @@ def test_huge_field_load(self): ) self.assertCML(cube, ("nimrod", f"{datafile}.cml")) + @tests.skip_data + def test_load_kwarg(self): + """Tests that the handle_metadata_errors kwarg is effective by setting it to + False with a file with known incomplete meta-data (missing ellipsoid).""" + datafile = "u1096_ng_ek00_pressure_2km" + with self.assertRaisesRegex( + TranslationError, + "Ellipsoid not supported, proj_biaxial_ellipsoid:-32767, horizontal_grid_type:0", + ): + iris.load( + tests.get_data_path(("NIMROD", "uk2km", "cutouts", datafile,)), + handle_metadata_errors=False, + ) + def test_orography(self): # Mock an orography field we've seen. field = mock_nimrod_field() From e8c983be7ee830a1480b95877c00e9ae70371569 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 28 May 2020 14:58:34 +0100 Subject: [PATCH 45/46] Dials back a couple of bits of functionality: - handle_metadata_errors is no longer a kwarg via iris.load_cube as the functionality is not in master yet and wasn't coded correctly here anyway - Removes "_above_threshold" from cloud field id 161 - Removing of zero-length bounds has been added but disabled until the merge_cube method stops stripping bounds where one cube has bounds=None. --- lib/iris/fileformats/nimrod.py | 9 ++------- lib/iris/fileformats/nimrod_load_rules.py | 8 ++++++-- lib/iris/tests/results/nimrod/probability_fields.cml | 8 ++++---- .../tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml | 2 +- lib/iris/tests/test_nimrod.py | 9 +++++---- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/iris/fileformats/nimrod.py b/lib/iris/fileformats/nimrod.py index 554b6e1a1d..2f4e32ed95 100644 --- a/lib/iris/fileformats/nimrod.py +++ b/lib/iris/fileformats/nimrod.py @@ -294,7 +294,7 @@ def _read_data(self, infile): self.data = self.data.reshape(self.num_rows, self.num_cols) -def load_cubes(filenames, callback=None, handle_metadata_errors=True): +def load_cubes(filenames, callback=None): """ Loads cubes from a list of NIMROD filenames. @@ -307,9 +307,6 @@ def load_cubes(filenames, callback=None, handle_metadata_errors=True): * callback - a function which can be passed on to :func:`iris.io.run_callback` - * handle_metadata_errors - Set to False to omit handling of known meta-data deficiencies - in Nimrod-format data - .. note:: The resultant cubes may not be in the same order as in the files. @@ -328,9 +325,7 @@ def load_cubes(filenames, callback=None, handle_metadata_errors=True): # End of file. Move on to the next file. break - cube = iris.fileformats.nimrod_load_rules.run( - field, handle_metadata_errors=handle_metadata_errors - ) + cube = iris.fileformats.nimrod_load_rules.run(field) # Were we given a callback? if callback is not None: diff --git a/lib/iris/fileformats/nimrod_load_rules.py b/lib/iris/fileformats/nimrod_load_rules.py index cb737bddaa..deb4ac862c 100644 --- a/lib/iris/fileformats/nimrod_load_rules.py +++ b/lib/iris/fileformats/nimrod_load_rules.py @@ -58,7 +58,7 @@ def name(cube, field, handle_metadata_errors): 102: "rain_melted_level_above_sea_level", 155: "Visibility", 156: "Worst visibility in grid point", - 161: "minimum_cloud_base_above_threshold", + 161: "minimum_cloud_base", 172: "cloud_area_fraction_in_atmosphere", 218: "snowfall", 421: "precipitation type", @@ -792,12 +792,16 @@ def probability_coord(cube, field, handle_metadata_errors): # If we found a coord_val, build the coord (with bounds) and add to cube) if coord_val is not None: - if field.threshold_fuzziness > -32766.0: + if not is_missing(field, field.threshold_fuzziness): bounds = [ coord_val * field.threshold_fuzziness, coord_val * (2.0 - field.threshold_fuzziness), ] bounds = np.array(bounds, dtype=np.float32) + # TODO: Enable filtering of zero-length bounds once Iris doesn't strip bounds + # in merge_cube + # if np.isclose(bounds[0], bounds[1]): + # bounds = None else: bounds = None if coord_keys.get("units", None) == "oktas": diff --git a/lib/iris/tests/results/nimrod/probability_fields.cml b/lib/iris/tests/results/nimrod/probability_fields.cml index eb54c785c6..7add3e75a4 100644 --- a/lib/iris/tests/results/nimrod/probability_fields.cml +++ b/lib/iris/tests/results/nimrod/probability_fields.cml @@ -297,7 +297,7 @@ - + @@ -337,7 +337,7 @@ - + @@ -384,7 +384,7 @@ - + @@ -428,7 +428,7 @@ - + diff --git a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml index 7253f93902..c6bc6f0419 100644 --- a/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml +++ b/lib/iris/tests/results/nimrod/u1096_ng_ek00_cloud_2km.cml @@ -248,7 +248,7 @@ - + diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 60076e378b..35055b61d3 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -86,10 +86,11 @@ def test_load_kwarg(self): TranslationError, "Ellipsoid not supported, proj_biaxial_ellipsoid:-32767, horizontal_grid_type:0", ): - iris.load( - tests.get_data_path(("NIMROD", "uk2km", "cutouts", datafile,)), - handle_metadata_errors=False, - ) + with open(tests.get_data_path(("NIMROD", "uk2km", "cutouts", datafile,)), "rb") as infile: + iris.fileformats.nimrod_load_rules.run( + iris.fileformats.nimrod.NimrodField(infile), + handle_metadata_errors=False, + ) def test_orography(self): # Mock an orography field we've seen. From 3a117ce7202bad7ea6aa7205f35b9bac8b862972 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Thu, 28 May 2020 17:39:33 +0100 Subject: [PATCH 46/46] Blacker code --- lib/iris/tests/test_nimrod.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/iris/tests/test_nimrod.py b/lib/iris/tests/test_nimrod.py index 35055b61d3..bcb9255dca 100644 --- a/lib/iris/tests/test_nimrod.py +++ b/lib/iris/tests/test_nimrod.py @@ -86,7 +86,10 @@ def test_load_kwarg(self): TranslationError, "Ellipsoid not supported, proj_biaxial_ellipsoid:-32767, horizontal_grid_type:0", ): - with open(tests.get_data_path(("NIMROD", "uk2km", "cutouts", datafile,)), "rb") as infile: + with open( + tests.get_data_path(("NIMROD", "uk2km", "cutouts", datafile,)), + "rb", + ) as infile: iris.fileformats.nimrod_load_rules.run( iris.fileformats.nimrod.NimrodField(infile), handle_metadata_errors=False,