From 830dbd7fb7fd67d2e501959d9704af7ff907d432 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 08:12:35 -0500 Subject: [PATCH 1/7] Fix loading composite configs when reader supports multiple sensors If a reader supports multiple sensors and there are composite configs for each sensor that conflict then there is no telling which one will get loaded first. This allows file handlers to tell the reader what sensor these files belong to. --- satpy/readers/file_handlers.py | 5 +++++ satpy/readers/geocat.py | 5 +++++ satpy/readers/yaml_reader.py | 22 +++++++++++++++++++++- satpy/tests/reader_tests/test_geocat.py | 2 +- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/satpy/readers/file_handlers.py b/satpy/readers/file_handlers.py index 9143463753..26fcf75b8b 100644 --- a/satpy/readers/file_handlers.py +++ b/satpy/readers/file_handlers.py @@ -114,3 +114,8 @@ def start_time(self): @property def end_time(self): return self.filename_info.get('end_time', self.start_time) + + @property + def sensor_names(self): + """List of sensors represented in this file.""" + raise NotImplementedError diff --git a/satpy/readers/geocat.py b/satpy/readers/geocat.py index 3dcc06c7a6..717c762f56 100644 --- a/satpy/readers/geocat.py +++ b/satpy/readers/geocat.py @@ -101,6 +101,10 @@ def _get_proj(self, platform, ref_lon): ref_lon = -75. return GEO_PROJS[platform].format(lon_0=ref_lon) + @property + def sensor_names(self): + return [self.get_sensor(self['/attr/Sensor_Name'])] + @property def start_time(self): return self.filename_info['start_time'] @@ -232,6 +236,7 @@ def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None data = data * factor + offset data.attrs.update(info) + data = data.rename({'lines': 'y', 'elements': 'x'}) return data diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index d191a5dbbe..b9a50cf3e3 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -104,12 +104,15 @@ def __init__(self, config_files): if not isinstance(self.info['sensors'], (list, tuple)): self.info['sensors'] = [self.info['sensors']] - self.sensor_names = self.info['sensors'] self.datasets = self.config.get('datasets', {}) self.info['filenames'] = [] self.ids = {} self.load_ds_ids_from_config() + @property + def sensor_names(self): + return self.info['sensors'] + @property def all_dataset_ids(self): return self.ids.keys() @@ -396,6 +399,23 @@ def __init__(self, logger.warning("Unrecognized/unused reader keyword argument(s) '{}'".format(kwargs)) self.coords_cache = WeakValueDictionary() + @property + def sensor_names(self): + if not self.file_handlers: + return self.info['sensors'] + + file_handlers = (handlers[0] for handlers in + self.file_handlers.values()) + sensor_names = set() + for fh in file_handlers: + try: + sensor_names.update(fh.sensor_names) + except NotImplementedError: + continue + if not sensor_names: + return self.info['sensors'] + return sorted(sensor_names) + @property def available_dataset_ids(self): for ds_id in self.all_dataset_ids: diff --git a/satpy/tests/reader_tests/test_geocat.py b/satpy/tests/reader_tests/test_geocat.py index 2853bcf391..9c48cb7341 100644 --- a/satpy/tests/reader_tests/test_geocat.py +++ b/satpy/tests/reader_tests/test_geocat.py @@ -96,7 +96,7 @@ def get_test_content(self, filename, filename_info, filetype_info): if key + '/attr/' + a in file_content: attrs[a] = file_content[key + '/attr/' + a] if val.ndim > 1: - file_content[key] = DataArray(val, dims=('y', 'x'), attrs=attrs) + file_content[key] = DataArray(val, dims=('lines', 'elements'), attrs=attrs) else: file_content[key] = DataArray(val, attrs=attrs) From 1ac2f79f277c1f2a1f4eb1791aa6831bf4a1528d Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 08:14:32 -0500 Subject: [PATCH 2/7] Fix AHI composites to work with native resampling (no reducers) --- satpy/composites/ahi.py | 141 +------------------- satpy/etc/composites/ahi.yaml | 176 ++++++++----------------- satpy/etc/readers/geocat.yaml | 240 +++++++++++++++++++++++++++++++++- 3 files changed, 300 insertions(+), 257 deletions(-) diff --git a/satpy/composites/ahi.py b/satpy/composites/ahi.py index 629f5388ed..34d7303022 100644 --- a/satpy/composites/ahi.py +++ b/satpy/composites/ahi.py @@ -24,149 +24,22 @@ import logging -import numpy as np -from pyresample.geometry import AreaDefinition -from satpy.composites import CompositeBase -from satpy.dataset import Dataset +from satpy.composites import GenericCompositor LOG = logging.getLogger(__name__) -class GreenCorrector(CompositeBase): +class GreenCorrector(GenericCompositor): """Corrector of the AHI green band to compensate for the deficit of chlorophyl signal. """ - def __call__(self, projectables, optional_datasets=None, **info): + def __call__(self, projectables, optional_datasets=None, **attrs): """Boost vegetation effect thanks to NIR (0.8µm) band.""" - (green, nir) = projectables - + green, nir = self.check_areas(projectables) LOG.info('Boosting vegetation on green band') - proj = Dataset(green * 0.85 + nir * 0.15, - copy=False, - **green.info) - self.apply_modifier_info(green, proj) - - return proj - - -class Expander(CompositeBase): - """Expand the size of the composite. - - Keyword Args: - factor (int): Repeat both dimensions by this number - - """ - def __call__(self, projectables, optional_datasets=None, **info): - factor = self.attrs.get('factor', 2) - - (band,) = projectables - - LOG.info('Expanding datasize by a factor %d.', factor) - - proj = Dataset( - np.repeat(np.repeat(band, factor, axis=0), factor, axis=1), - copy=False, **band.info) - - old_area = proj.info['area'] - proj.info['area'] = AreaDefinition(old_area.area_id, - old_area.name, - old_area.proj_id, - old_area.proj_dict, - old_area.x_size * factor, - old_area.y_size * factor, - old_area.area_extent) - proj.info['resolution'] *= factor - self.apply_modifier_info(band, proj) - return proj - - -class Reducer2(CompositeBase): - """Reduce the size of the composite.""" - - def __call__(self, projectables, optional_datasets=None, **info): - (band,) = projectables - - factor = 2 - - LOG.info('Reducing datasize by a factor %d.', factor) - - proj = band[::factor, ::factor] - # newshape = (band.shape[0] / factor, factor, - # band.shape[1] / factor, factor) - # proj = Dataset(band.reshape(newshape).mean(axis=3).mean(axis=1), - # copy=False, **band.info) - - old_area = proj.attrs['area'] - proj.attrs['area'] = AreaDefinition(old_area.area_id, - old_area.name, - old_area.proj_id, - old_area.proj_dict, - old_area.x_size / factor, - old_area.y_size / factor, - old_area.area_extent) - proj.attrs['resolution'] *= factor - self.apply_modifier_info(band, proj) - return proj - - -class Reducer4(CompositeBase): - """Reduce the size of the composite.""" - - def __call__(self, projectables, optional_datasets=None, **info): - (band,) = projectables - - factor = 4 - - LOG.info('Reducing datasize by a factor %d.', factor) - - proj = band[::factor, ::factor] - - # newshape = (band.shape[0] / factor, factor, - # band.shape[1] / factor, factor) - # proj = Dataset(band.reshape(newshape).mean(axis=3).mean(axis=1), - # copy=False, **band.info) - - old_area = proj.attrs['area'] - proj.attrs['area'] = AreaDefinition(old_area.area_id, - old_area.name, - old_area.proj_id, - old_area.proj_dict, - old_area.x_size / factor, - old_area.y_size / factor, - old_area.area_extent) - proj.attrs['resolution'] *= factor - self.apply_modifier_info(band, proj) - return proj - - -class Reducer8(CompositeBase): - """Reduce the size of the composite.""" - - def __call__(self, projectables, optional_datasets=None, **info): - (band,) = projectables - - factor = 8 - - LOG.info('Reducing datasize by a factor %d.', factor) - - proj = band[::factor, ::factor] - - # newshape = (band.shape[0] / factor, factor, - # band.shape[1] / factor, factor) - # proj = Dataset(band.reshape(newshape).mean(axis=3).mean(axis=1), - # copy=False, **band.info) - - old_area = proj.attrs['area'] - proj.attrs['area'] = AreaDefinition(old_area.area_id, - old_area.name, - old_area.proj_id, - old_area.proj_dict, - old_area.x_size / factor, - old_area.y_size / factor, - old_area.area_extent) - proj.attrs['resolution'] *= factor - self.apply_modifier_info(band, proj) - return proj + new_green = green * 0.85 + nir * 0.15 + new_green.attrs = green.attrs.copy() + return super(GreenCorrector, self).__call__((new_green,), **attrs) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 80ad021dfb..8d1ebcc997 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -1,76 +1,33 @@ sensor_name: visir/ahi modifiers: - reducer2: - compositor: !!python/name:satpy.composites.ahi.Reducer2 - - reducer4: - compositor: !!python/name:satpy.composites.ahi.Reducer4 - - vegetation_corrected: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector - prerequisites: - - wavelength: 0.85 - - vegetation_corrected_reduced: - compositor: !!python/name:satpy.composites.ahi.GreenCorrector - prerequisites: - - wavelength: 0.85 - modifiers: [reducer2,] - rayleigh_corrected: compositor: !!python/name:satpy.composites.PSPRayleighReflectance atmosphere: us-standard aerosol_type: marine_clean_aerosol prerequisites: - wavelength: 0.65 - modifiers: [reducer2, sunz_corrected] - optional_prerequisites: - - satellite_azimuth_angle - - satellite_zenith_angle - - solar_azimuth_angle - - solar_zenith_angle - - rayleigh_corrected_reducedsize: - compositor: !!python/name:satpy.composites.PSPRayleighReflectance - atmosphere: us-standard - aerosol_type: marine_clean_aerosol - prerequisites: - - wavelength: 0.65 - modifiers: [reducer4, sunz_corrected] + modifiers: [sunz_corrected] optional_prerequisites: - satellite_azimuth_angle - satellite_zenith_angle - solar_azimuth_angle - solar_zenith_angle - rayleigh_corrected_reducedsize_land: - compositor: !!python/name:satpy.composites.PSPRayleighReflectance - atmosphere: us-standard - aerosol_type: continental_average_aerosol - prerequisites: - - wavelength: 0.65 - modifiers: [reducer4, sunz_corrected] - optional_prerequisites: - - satellite_azimuth_angle - - satellite_zenith_angle - - solar_azimuth_angle - - solar_zenith_angle - - rayleigh_corrected_reducedsize_marine_tropical: - compositor: !!python/name:satpy.composites.PSPRayleighReflectance - atmosphere: us-standard - aerosol_type: marine_tropical_aerosol +composites: + green: + compositor: !!python/name:satpy.composites.ahi.GreenCorrector + # FUTURE: Set a wavelength...see what happens. Dependency finding + # probably wouldn't work. prerequisites: - - wavelength: 0.65 - modifiers: [reducer4, sunz_corrected] - optional_prerequisites: - - satellite_azimuth_angle - - satellite_zenith_angle - - solar_azimuth_angle - - solar_zenith_angle + # should we be using the most corrected or least corrected inputs? + # what happens if something requests more modifiers on top of this? + - wavelength: 0.51 + modifiers: [sunz_corrected] + - wavelength: 0.85 + modifiers: [sunz_corrected] + standard_name: toa_bidirection_reflectance -composites: overview: compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: @@ -79,90 +36,71 @@ composites: - 10.4 standard_name: overview - true_color: - compositor: !!python/name:satpy.composites.GenericCompositor + natural: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB prerequisites: - - wavelength: 0.65 - modifiers: [reducer2, effective_solar_pathlength_corrected, rayleigh_corrected] - - wavelength: 0.51 - modifiers: [vegetation_corrected, effective_solar_pathlength_corrected, rayleigh_corrected] - - wavelength: 0.46 - modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] - standard_name: true_color - - true_color_reducedsize: - compositor: !!python/name:satpy.composites.GenericCompositor - prerequisites: - - wavelength: 0.65 - modifiers: [reducer4, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize] - - wavelength: 0.51 - modifiers: [reducer2, vegetation_corrected_reduced, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize] - - wavelength: 0.46 - modifiers: [reducer2, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize] - standard_name: true_color - - true_color_reducedsize_land: - compositor: !!python/name:satpy.composites.GenericCompositor - prerequisites: - - wavelength: 0.65 - modifiers: [reducer4, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize_land] - - wavelength: 0.51 - modifiers: [reducer2, vegetation_corrected_reduced, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize_land] - - wavelength: 0.46 - modifiers: [reducer2, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize_land] - standard_name: true_color + - wavelength: 1.63 + modifiers: [sunz_corrected] #, rayleigh_corrected] + - wavelength: 0.85 + modifiers: [sunz_corrected] #, rayleigh_corrected] + - wavelength: 0.635 + modifiers: [sunz_corrected] #, rayleigh_corrected] + high_resolution_band: blue + standard_name: natural - true_color_reducedsize_marine_tropical: + true_color: compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: - wavelength: 0.65 - modifiers: [reducer4, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize_marine_tropical] - - wavelength: 0.51 - modifiers: [reducer2, vegetation_corrected_reduced, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize_marine_tropical] + modifiers: [sunz_corrected] #, rayleigh_corrected] + - name: green - wavelength: 0.46 - modifiers: [reducer2, effective_solar_pathlength_corrected, - rayleigh_corrected_reducedsize_marine_tropical] + modifiers: [sunz_corrected] #, rayleigh_corrected] standard_name: true_color - true_color_norayleigh: - compositor: !!python/name:satpy.composites.GenericCompositor - prerequisites: - - wavelength: 0.65 - modifiers: [reducer2, effective_solar_pathlength_corrected] - - wavelength: 0.51 - modifiers: [vegetation_corrected, effective_solar_pathlength_corrected] - - wavelength: 0.46 - modifiers: [effective_solar_pathlength_corrected] - standard_name: true_color +# true_color_reducedsize_land: +# compositor: !!python/name:satpy.composites.GenericCompositor +# prerequisites: +# - wavelength: 0.65 +# modifiers: [reducer4, effective_solar_pathlength_corrected, +# rayleigh_corrected_reducedsize_land] +# - wavelength: 0.51 +# modifiers: [reducer2, vegetation_corrected_reduced, effective_solar_pathlength_corrected, +# rayleigh_corrected_reducedsize_land] +# - wavelength: 0.46 +# modifiers: [reducer2, effective_solar_pathlength_corrected, +# rayleigh_corrected_reducedsize_land] +# standard_name: true_color +# +# true_color_reducedsize_marine_tropical: +# compositor: !!python/name:satpy.composites.GenericCompositor +# prerequisites: +# - wavelength: 0.65 +# modifiers: [reducer4, effective_solar_pathlength_corrected, +# rayleigh_corrected_reducedsize_marine_tropical] +# - wavelength: 0.51 +# modifiers: [reducer2, vegetation_corrected_reduced, effective_solar_pathlength_corrected, +# rayleigh_corrected_reducedsize_marine_tropical] +# - wavelength: 0.46 +# modifiers: [reducer2, effective_solar_pathlength_corrected, +# rayleigh_corrected_reducedsize_marine_tropical] +# standard_name: true_color day_microphysics_eum: compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: - wavelength: 0.86 - modifiers: [reducer4, ] - wavelength: 3.9 - modifiers: [nir_reflectance, reducer2] + modifiers: [nir_reflectance] - wavelength: 10.4 - modifiers: [reducer2, ] standard_name: day_microphysics day_microphysics_ahi: compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: - wavelength: 0.86 - modifiers: [reducer4, ] - wavelength: 2.3 - modifiers: [reducer2] - wavelength: 10.4 - modifiers: [reducer2, ] standard_name: day_microphysics cloud_phase_distinction: @@ -170,7 +108,6 @@ composites: prerequisites: - wavelength: 10.4 - wavelength: 0.64 - modifiers: [reducer4] - wavelength: 1.6 standard_name: cloud_phase_distinction @@ -200,8 +137,7 @@ composites: convection: compositor: !!python/name:satpy.composites.Convection prerequisites: - - wavelength: 0.635 - modifiers: [reducer4] + - 0.635 - 1.63 - 3.75 - 6.7 diff --git a/satpy/etc/readers/geocat.yaml b/satpy/etc/readers/geocat.yaml index b9e0757462..fa9aa6ad3e 100644 --- a/satpy/etc/readers/geocat.yaml +++ b/satpy/etc/readers/geocat.yaml @@ -1,5 +1,5 @@ reader: - description: GEOCAT + description: CSPP Geo and GEOCAT file reader name: geocat reader: !!python/name:satpy.readers.geocat.GEOCATYAMLReader sensors: [abi, ahi, goes_imager] @@ -11,9 +11,243 @@ file_types: - 'geocatL{processing_level:1d}.{platform_shortname}.{start_time:%Y%j.%H%M%S}.hdf' - 'geocatL{processing_level:1d}.{platform_shortname}.{start_time:%Y%j.%H%M%S}.nc' # Himawari 8 files: - - 'geocatL{processing_level:1d}.{platform_shortname}.{start_time:%Y%j.%H%M%S}.{sector_id}.{res_id}.hdf' - - 'geocatL{processing_level:1d}.{platform_shortname}.{start_time:%Y%j.%H%M%S}.{sector_id}.{res_id}.nc' + - 'geocatL2.{platform_shortname}.{start_time:%Y%j.%H%M%S}.{sector_id}.{res_id}.hdf' + - 'geocatL2.{platform_shortname}.{start_time:%Y%j.%H%M%S}.{sector_id}.{res_id}.nc' # GOES-16 ABI files: - 'geocatL{processing_level:1d}.{platform_shortname}.{sector_id}.{start_time:%Y%j.%H%M%S}.hdf' - 'geocatL{processing_level:1d}.{platform_shortname}.{sector_id}.{start_time:%Y%j.%H%M%S}.nc' + ahi_level1: + file_reader: !!python/name:satpy.readers.geocat.GEOCATFileHandler + file_patterns: + # we could use the H8 pattern above, but then the datasets listed below + # would always be "available" + - 'geocatL1.HIMAWARI-8.{start_time:%Y%j.%H%M%S}.{sector_id}.{res_id}.hdf' + - 'geocatL1.HIMAWARI-8.{start_time:%Y%j.%H%M%S}.{sector_id}.{res_id}.nc' +datasets: + # AHI Level 1 Datasets (need to define here so wavelengths can be used) + B01: + name: B01 + sensor: ahi + wavelength: [0.45,0.47,0.49] + # FIXME: The resolution depends on the input files + resolution: 1000 + calibration: + reflectance: + file_key: himawari_8_ahi_channel_1_reflectance + standard_name: toa_bidirectional_reflectance + units: "%" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B02: + name: B02 + sensor: ahi + wavelength: [0.49,0.51,0.53] + resolution: 1000 + calibration: + reflectance: + file_key: himawari_8_ahi_channel_2_reflectance + standard_name: toa_bidirectional_reflectance + units: "%" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B03: + name: B03 + sensor: ahi + wavelength: [0.62,0.64,0.66] + resolution: 500 + calibration: + reflectance: + file_key: himawari_8_ahi_channel_3_reflectance + standard_name: toa_bidirectional_reflectance + units: "%" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B04: + name: B04 + sensor: ahi + wavelength: [0.83, 0.85, 0.87] + resolution: 1000 + calibration: + reflectance: + file_key: himawari_8_ahi_channel_4_reflectance + standard_name: toa_bidirectional_reflectance + units: "%" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B05: + name: B05 + sensor: ahi + wavelength: [1.5, 1.6, 1.7] + resolution: 2000 + calibration: + reflectance: + file_key: himawari_8_ahi_channel_5_reflectance + standard_name: toa_bidirectional_reflectance + units: "%" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B06: + name: B06 + sensor: ahi + wavelength: [2.2, 2.3, 2.4] + resolution: 2000 + calibration: + reflectance: + file_key: himawari_8_ahi_channel_6_reflectance + standard_name: toa_bidirectional_reflectance + units: "%" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B07: + name: B07 + sensor: ahi + wavelength: [3.7, 3.9, 4.1] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_7_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B08: + name: B08 + sensor: ahi + wavelength: [6.0, 6.2, 6.4] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_8_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B09: + name: B09 + sensor: ahi + wavelength: [6.7, 6.9, 7.1] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_9_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B10: + name: B10 + sensor: ahi + wavelength: [7.1, 7.3, 7.5] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_10_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B11: + name: B11 + sensor: ahi + wavelength: [8.4, 8.6, 8.8] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_11_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B12: + name: B12 + sensor: ahi + wavelength: [9.4, 9.6, 9.8] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_12_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B13: + name: B13 + sensor: ahi + wavelength: [10.2, 10.4, 10.6] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_13_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B14: + name: B14 + sensor: ahi + wavelength: [11.0, 11.2, 11.4] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_14_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B15: + name: B15 + sensor: ahi + wavelength: [12.2, 12.4, 12.6] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_15_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 + B16: + name: B16 + sensor: ahi + wavelength: [13.1, 13.3, 13.5] + resolution: 2000 + calibration: + brightness_temperature: + file_key: himawari_8_ahi_channel_16_brightness_temperature + standard_name: toa_brightness_temperature + units: "K" +# radiance: +# standard_name: toa_outgoing_radiance_per_unit_wavelength +# units: W m-2 um-1 sr-1 + file_type: ahi_level1 \ No newline at end of file From a66b8031521bf0604fa487a87eec0dc637be9401 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 08:36:01 -0500 Subject: [PATCH 3/7] Change AHI true color to self-sharpened RGB --- satpy/etc/composites/ahi.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 8d1ebcc997..f3255322d4 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -49,13 +49,14 @@ composites: standard_name: natural true_color: - compositor: !!python/name:satpy.composites.GenericCompositor + compositor: !!python/name:satpy.composites.SelfSharpenedRGB prerequisites: - wavelength: 0.65 modifiers: [sunz_corrected] #, rayleigh_corrected] - name: green - wavelength: 0.46 modifiers: [sunz_corrected] #, rayleigh_corrected] + high_resolution_band: red standard_name: true_color # true_color_reducedsize_land: From e2b96f56fcd22ec5adf503b73c9dc9703c5c7b65 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 08:36:20 -0500 Subject: [PATCH 4/7] Add AHI composite tests --- satpy/tests/compositor_tests/__init__.py | 3 +- satpy/tests/compositor_tests/test_abi.py | 6 +- satpy/tests/compositor_tests/test_ahi.py | 75 ++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 satpy/tests/compositor_tests/test_ahi.py diff --git a/satpy/tests/compositor_tests/__init__.py b/satpy/tests/compositor_tests/__init__.py index b3a169a5c4..1e6b6fce80 100644 --- a/satpy/tests/compositor_tests/__init__.py +++ b/satpy/tests/compositor_tests/__init__.py @@ -22,7 +22,7 @@ import sys -from satpy.tests.compositor_tests import test_abi +from satpy.tests.compositor_tests import test_abi, test_ahi if sys.version_info < (2, 7): import unittest2 as unittest @@ -119,6 +119,7 @@ def suite(): loader = unittest.TestLoader() mysuite = unittest.TestSuite() mysuite.addTests(test_abi.suite()) + mysuite.addTests(test_ahi.suite()) mysuite.addTest(loader.loadTestsFromTestCase(TestCheckArea)) return mysuite diff --git a/satpy/tests/compositor_tests/test_abi.py b/satpy/tests/compositor_tests/test_abi.py index 38ce051a8a..f3aae8c782 100644 --- a/satpy/tests/compositor_tests/test_abi.py +++ b/satpy/tests/compositor_tests/test_abi.py @@ -28,7 +28,11 @@ class TestABIComposites(unittest.TestCase): + + """Test ABI-specific composites.""" + def test_simulated_green(self): + """Test creating a fake 'green' band.""" import xarray as xr import dask.array as da import numpy as np @@ -65,7 +69,7 @@ def test_simulated_green(self): def suite(): - """The test suite for test_scene. + """The test suite for test_abi. """ loader = unittest.TestLoader() mysuite = unittest.TestSuite() diff --git a/satpy/tests/compositor_tests/test_ahi.py b/satpy/tests/compositor_tests/test_ahi.py new file mode 100644 index 0000000000..81c7721204 --- /dev/null +++ b/satpy/tests/compositor_tests/test_ahi.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2018 PyTroll developers +# +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Tests for AHI compositors. +""" + +import sys + +if sys.version_info < (2, 7): + import unittest2 as unittest +else: + import unittest + + +class TestAHIComposites(unittest.TestCase): + + """Test AHI-specific composites.""" + + def test_corrected_green(self): + """Test adjusting the 'green' band.""" + import xarray as xr + import dask.array as da + import numpy as np + from satpy.composites.ahi import GreenCorrector + from pyresample.geometry import AreaDefinition + rows = 5 + cols = 10 + area = AreaDefinition( + 'test', 'test', 'test', + {'proj': 'eqc', 'lon_0': 0.0, + 'lat_0': 0.0}, + cols, rows, + (-20037508.34, -10018754.17, 20037508.34, 10018754.17)) + + comp = GreenCorrector('green', prerequisites=(0.51, 0.85), + standard_name='toa_bidirectional_reflectance') + c01 = xr.DataArray(da.zeros((rows, cols), chunks=25) + 0.25, + dims=('y', 'x'), + attrs={'name': 'C01', 'area': area}) + c02 = xr.DataArray(da.zeros((rows, cols), chunks=25) + 0.30, + dims=('y', 'x'), + attrs={'name': 'C02', 'area': area}) + res = comp((c01, c02)) + self.assertIsInstance(res, xr.DataArray) + self.assertIsInstance(res.data, da.Array) + self.assertEqual(res.attrs['name'], 'green') + self.assertEqual(res.attrs['standard_name'], + 'toa_bidirectional_reflectance') + data = res.compute() + print(data) + np.testing.assert_allclose(data, 0.28025) + + +def suite(): + """The test suite for test_ahi. + """ + loader = unittest.TestLoader() + mysuite = unittest.TestSuite() + mysuite.addTest(loader.loadTestsFromTestCase(TestAHIComposites)) + return mysuite From e5988bf8d11de69b0b5e7fb50f26b8e3efe5a710 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 08:38:32 -0500 Subject: [PATCH 5/7] Fix AHI composite test (oops) --- satpy/tests/compositor_tests/test_ahi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/satpy/tests/compositor_tests/test_ahi.py b/satpy/tests/compositor_tests/test_ahi.py index 81c7721204..93d300fa0e 100644 --- a/satpy/tests/compositor_tests/test_ahi.py +++ b/satpy/tests/compositor_tests/test_ahi.py @@ -62,8 +62,7 @@ def test_corrected_green(self): self.assertEqual(res.attrs['standard_name'], 'toa_bidirectional_reflectance') data = res.compute() - print(data) - np.testing.assert_allclose(data, 0.28025) + np.testing.assert_allclose(data, 0.2575) def suite(): From 2fb75b414c8d3ff799083dc4207d898c3ffa5d03 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 09:30:49 -0500 Subject: [PATCH 6/7] Fix geocat resolution handling to give human friendly ID resolutions --- satpy/etc/readers/geocat.yaml | 17 ----------------- satpy/readers/geocat.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/satpy/etc/readers/geocat.yaml b/satpy/etc/readers/geocat.yaml index fa9aa6ad3e..f93a3d8524 100644 --- a/satpy/etc/readers/geocat.yaml +++ b/satpy/etc/readers/geocat.yaml @@ -30,8 +30,6 @@ datasets: name: B01 sensor: ahi wavelength: [0.45,0.47,0.49] - # FIXME: The resolution depends on the input files - resolution: 1000 calibration: reflectance: file_key: himawari_8_ahi_channel_1_reflectance @@ -45,7 +43,6 @@ datasets: name: B02 sensor: ahi wavelength: [0.49,0.51,0.53] - resolution: 1000 calibration: reflectance: file_key: himawari_8_ahi_channel_2_reflectance @@ -59,7 +56,6 @@ datasets: name: B03 sensor: ahi wavelength: [0.62,0.64,0.66] - resolution: 500 calibration: reflectance: file_key: himawari_8_ahi_channel_3_reflectance @@ -73,7 +69,6 @@ datasets: name: B04 sensor: ahi wavelength: [0.83, 0.85, 0.87] - resolution: 1000 calibration: reflectance: file_key: himawari_8_ahi_channel_4_reflectance @@ -87,7 +82,6 @@ datasets: name: B05 sensor: ahi wavelength: [1.5, 1.6, 1.7] - resolution: 2000 calibration: reflectance: file_key: himawari_8_ahi_channel_5_reflectance @@ -101,7 +95,6 @@ datasets: name: B06 sensor: ahi wavelength: [2.2, 2.3, 2.4] - resolution: 2000 calibration: reflectance: file_key: himawari_8_ahi_channel_6_reflectance @@ -115,7 +108,6 @@ datasets: name: B07 sensor: ahi wavelength: [3.7, 3.9, 4.1] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_7_brightness_temperature @@ -129,7 +121,6 @@ datasets: name: B08 sensor: ahi wavelength: [6.0, 6.2, 6.4] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_8_brightness_temperature @@ -143,7 +134,6 @@ datasets: name: B09 sensor: ahi wavelength: [6.7, 6.9, 7.1] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_9_brightness_temperature @@ -157,7 +147,6 @@ datasets: name: B10 sensor: ahi wavelength: [7.1, 7.3, 7.5] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_10_brightness_temperature @@ -171,7 +160,6 @@ datasets: name: B11 sensor: ahi wavelength: [8.4, 8.6, 8.8] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_11_brightness_temperature @@ -185,7 +173,6 @@ datasets: name: B12 sensor: ahi wavelength: [9.4, 9.6, 9.8] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_12_brightness_temperature @@ -199,7 +186,6 @@ datasets: name: B13 sensor: ahi wavelength: [10.2, 10.4, 10.6] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_13_brightness_temperature @@ -213,7 +199,6 @@ datasets: name: B14 sensor: ahi wavelength: [11.0, 11.2, 11.4] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_14_brightness_temperature @@ -227,7 +212,6 @@ datasets: name: B15 sensor: ahi wavelength: [12.2, 12.4, 12.6] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_15_brightness_temperature @@ -241,7 +225,6 @@ datasets: name: B16 sensor: ahi wavelength: [13.1, 13.3, 13.5] - resolution: 2000 calibration: brightness_temperature: file_key: himawari_8_ahi_channel_16_brightness_temperature diff --git a/satpy/readers/geocat.py b/satpy/readers/geocat.py index 717c762f56..9472c5f03f 100644 --- a/satpy/readers/geocat.py +++ b/satpy/readers/geocat.py @@ -74,6 +74,7 @@ class GEOCATFileHandler(NetCDF4FileHandler): 'ahi': { 1: 999.9999820317674, # assumption 2: 1999.999964063535, + 4: 3999.99992812707, } } @@ -118,11 +119,20 @@ def is_geo(self): platform = self.get_platform(self['/attr/Platform_Name']) return platform in GEO_PROJS - def available_dataset_ids(self): - """Automatically determine datasets provided by this file""" + @property + def resolution(self): elem_res = self['/attr/Element_Resolution'] + return int(elem_res * 1000) + + def _calc_area_resolution(self, ds_res): + elem_res = round(ds_res / 1000.) # mimic 'Element_Resolution' attribute from above sensor = self.get_sensor(self['/attr/Sensor_Name']) - res = self.resolutions.get(sensor, {}).get(int(elem_res), elem_res * 1000.) + return self.resolutions.get(sensor, {}).get(int(elem_res), + elem_res * 1000.) + + def available_dataset_ids(self): + """Automatically determine datasets provided by this file""" + res = self.resolution coordinates = ['pixel_longitude', 'pixel_latitude'] for var_name, val in self.file_content.items(): if isinstance(val, netCDF4.Variable): @@ -178,7 +188,7 @@ def get_area_def(self, dsid): raise NotImplementedError("Don't know how to get the Area Definition for this file") platform = self.get_platform(self['/attr/Platform_Name']) - res = dsid.resolution + res = self._calc_area_resolution(dsid.resolution) proj = self._get_proj(platform, float(self['/attr/Subsatellite_Longitude'])) area_name = '{} {} Area at {}m'.format( platform, @@ -248,8 +258,22 @@ def create_filehandlers(self, filenames): def load_ds_ids_from_files(self): for file_handlers in self.file_handlers.values(): fh = file_handlers[0] + # update resolution in the dataset IDs for this files resolution + res = fh.resolution + for ds_id, ds_info in list(self.ids.items()): + if fh.filetype_info['file_type'] != ds_info['file_type']: + continue + if ds_id.resolution is not None: + continue + ds_info['resolution'] = res + new_id = DatasetID.from_dict(ds_info) + self.ids[new_id] = ds_info + del self.ids[ds_id] + + # dynamically discover other available datasets for ds_id, ds_info in fh.available_dataset_ids(): # don't overwrite an existing dataset # especially from the yaml config self.ids.setdefault(ds_id, ds_info) + From c0be224e26b200a66f2639952f5a4facf81aeb97 Mon Sep 17 00:00:00 2001 From: davidh-ssec Date: Mon, 26 Mar 2018 09:31:26 -0500 Subject: [PATCH 7/7] Fix native resampler for arrays with more than 2 dimensions --- satpy/resample.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/satpy/resample.py b/satpy/resample.py index 5af73428ad..7a6ccd7db9 100644 --- a/satpy/resample.py +++ b/satpy/resample.py @@ -577,8 +577,8 @@ def compute(self, data, expand=True, **kwargs): out_shape = target_geo_def.shape in_shape = data.shape - y_repeats = out_shape[y_axis] / float(in_shape[y_axis]) - x_repeats = out_shape[x_axis] / float(in_shape[x_axis]) + y_repeats = out_shape[0] / float(in_shape[y_axis]) + x_repeats = out_shape[1] / float(in_shape[x_axis]) repeats = { y_axis: y_repeats, x_axis: x_repeats, @@ -597,6 +597,9 @@ def compute(self, data, expand=True, **kwargs): coords['y'] = y_coord if 'x' in data.coords: coords['x'] = x_coord + for dim in data.dims: + if dim not in ['y', 'x'] and dim in data.coords: + coords[dim] = data.coords[dim] return xr.DataArray(d_arr, dims=data.dims,