Skip to content

Commit

Permalink
Merge pull request #1503 from ColinDuff/seviri_l2_grib_reader_update
Browse files Browse the repository at this point in the history 
Fix issue with reading MSG GRIB products from the eumetsat datastore
  • Loading branch information
@mraspaud
mraspaud authored Jan 22, 2021
2 parents 93f0233 + 3ea7001 commit 2df64af
Show file tree
Hide file tree
Showing 2 changed files with 175 additions and 19 deletions.
183 changes: 169 additions & 14 deletions satpy/etc/readers/seviri_l2_grib.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,59 @@ file_types:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'CLMEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGCLMK-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCLMK-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCLMK-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCLMK-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCLMK-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

# EUMETSAT MSG SEVIRI L2 Optimal Cloud Analysis files in GRIB format
grib_seviri_oca:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'OCAEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGOCAE-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGOCAE-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGOCAE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGOCAE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGOCAE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

# EUMETSAT MSG SEVIRI L2 Active Fire Monitoring files in GRIB format
grib_seviri_fir:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'FIREncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGFIRG-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGFIRG-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGFIRG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGFIRG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGFIRG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_aes:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'AESGRIBProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGAESE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGAESE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGAESE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_cth:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'CTHEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGCLTH-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCLTH-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCLTH-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_crm:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'CRMEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGCRMN-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCRMN-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCRMN-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_mpe:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'MPEGRIBProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGMPEG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGMPEG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGMPEG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

datasets:

Expand All @@ -38,31 +73,31 @@ datasets:
resolution: 3000
file_type: grib_seviri_clm
parameter_number: 7
units: ""
units: "1"
long_name: cloud_mask_classification

pixel_scene_type:
name: pixel_scene_type
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 8
units: ""
units: "1"
long_name: scene_classification

measurement_cost:
name: measurement_cost
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 30
units: ""
units: "1"
long_name: cost_function

upper_layer_cloud_optical_depth:
name: upper_layer_cloud_optical_depth
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 31
units: ""
units: "1"
long_name: cloud_optical_depth

upper_layer_cloud_top_pressure:
Expand All @@ -86,7 +121,7 @@ datasets:
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 34
units: ""
units: "1"
long_name: cloud_optical_depth

error_in_upper_layer_cloud_top_pressure:
Expand All @@ -110,7 +145,7 @@ datasets:
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 37
units: ""
units: "1"
long_name: cloud_optical_depth

lower_layer_cloud_top_pressure:
Expand All @@ -126,7 +161,7 @@ datasets:
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 39
units: ""
units: "1"
long_name: cloud_optical_depth_standard_error

error_in_lower_layer_cloud_top_pressure:
Expand All @@ -150,5 +185,125 @@ datasets:
resolution: 3000
file_type: grib_seviri_fir
parameter_number: 9
units: ""
units: "1"
long_name: active_fire_classification

aerosol_optical_thickness_vis06:
name: aerosol_optical_thickness_vis06
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 20
units: "um"
long_name: aerosol_optical_thickness_vis06

aerosol_optical_thickness_vis08:
name: aerosol_optical_thickness_vis08
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 21
units: "um"
long_name: aerosol_optical_thickness_vis08

aerosol_optical_thickness_vis16:
name: aerosol_optical_thickness_vis16
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 22
units: "um"
long_name: aerosol_optical_thickness_vis06

angstroem_coefficient:
name: angstroem_coefficient
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 23
units: "1"
long_name: angstroem_coefficient

aes_quality:
name: aes_quality
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 192
units: "1"
long_name: aes_quality

cloud_top_height:
name: cloud_top_height
resolution: 3000
file_type: grib_seviri_cth
parameter_number: 2
units: Pa
long_name: cloud_top_height

cloud_top_quality:
name: cloud_top_quality
resolution: 3000
file_type: grib_seviri_cth
parameter_number: 3
units: "1"
long_name: cloud_top_quality

vis_refl_06:
name: vis_refl_06
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 9
units: "%"
long_name: vis_refl_06

vis_refl_08:
name: vis_refl_08
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 10
units: "%"
long_name: vis_refl_08

vis_refl_16:
name: vis_refl_16
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 11
units: "%"
long_name: vis_refl_16

nir_refl_39:
name: nir_refl_39
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 12
units: "%"
long_name: nir_refl_39

num_accumulations:
name: num_accumulations
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 6
units: "1"
long_name: num_accumulations

azimuth_angle:
name: azimuth_angle
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 7
units: degrees
long_name: azimuth_angle

relative_azimuth_angle:
name: relative_azimuth_angle
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 8
units: degrees
long_name: relative_azimuth_angle

instantaneous_rain_rate:
name: instantaneous_rain_rate
resolution: 3000
file_type: grib_seviri_mpe
parameter_number: 1
units: "kg m-2 s-1"
long_name: instantaneous_rain_rate
11 changes: 6 additions & 5 deletions satpy/readers/seviri_l2_grib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,17 @@
"""
import logging
import dask.array as da
import numpy as np
import xarray as xr
import dask.array as da

from datetime import timedelta

from satpy.readers.file_handlers import BaseFileHandler
from satpy import CHUNK_SIZE
from satpy.readers._geos_area import get_area_definition
from satpy.readers.file_handlers import BaseFileHandler
from satpy.readers.seviri_base import (calculate_area_extent,
PLATFORM_DICT,
REPEAT_CYCLE_DURATION)
from satpy import CHUNK_SIZE

try:
import eccodes as ec
Expand Down Expand Up @@ -143,6 +142,7 @@ def get_dataset(self, dataset_id, dataset_info):

# Check if the parameter number in the GRIB message corresponds to the required key
parameter_number = self._get_from_msg(gid, 'parameterNumber')

if parameter_number != dataset_info['parameter_number']:
# The parameter number is not the correct one, skip to next message
ec.codes_release(gid)
Expand All @@ -153,7 +153,8 @@ def get_dataset(self, dataset_id, dataset_info):

# Retrieve values and metadata from the GRIB message, masking the values equal to missing_value
xarr = self._get_xarray_from_msg(gid)
xarr.where(xarr.data == missing_value, np.NaN)

xarr.data = da.where(xarr.data == missing_value, np.nan, xarr.data)

ec.codes_release(gid)

Expand Down

0 comments on commit 2df64af

Please sign in to comment.