From 6fd553b191036d6fb287d9397e62cadc3128d084 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 1 Oct 2018 12:00:52 +0200 Subject: [PATCH 01/13] Allow on-the-fly decompression of xRIT files in xRIT readers --- satpy/etc/readers/hrit_electrol.yaml | 4 +-- satpy/etc/readers/hrit_goes.yaml | 8 ++--- satpy/etc/readers/hrit_msg.yaml | 4 +-- satpy/readers/hrit_base.py | 54 +++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/satpy/etc/readers/hrit_electrol.yaml b/satpy/etc/readers/hrit_electrol.yaml index 32915fdfc5..7593273cfa 100644 --- a/satpy/etc/readers/hrit_electrol.yaml +++ b/satpy/etc/readers/hrit_electrol.yaml @@ -58,11 +58,11 @@ file_types: HRIT_PRO_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSPrologueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-_________-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-_________-PRO______-{start_time:%Y%m%d%H%M}-__'] HRIT_EPI_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSEpilogueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-_________-EPI______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-_________-EPI______-{start_time:%Y%m%d%H%M}-__'] datasets: '00_6': diff --git a/satpy/etc/readers/hrit_goes.yaml b/satpy/etc/readers/hrit_goes.yaml index 0e1f26a28e..422e96ddfc 100644 --- a/satpy/etc/readers/hrit_goes.yaml +++ b/satpy/etc/readers/hrit_goes.yaml @@ -32,19 +32,19 @@ file_types: HRIT_PRO_00: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-__'] HRIT_PRO_03: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-__'] HRIT_PRO_06: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-__'] HRIT_PRO_10: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESPrologueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-PRO______-{start_time:%Y%m%d%H%M}-__'] datasets: '00_7': diff --git a/satpy/etc/readers/hrit_msg.yaml b/satpy/etc/readers/hrit_msg.yaml index f868167f02..e3770603be 100644 --- a/satpy/etc/readers/hrit_msg.yaml +++ b/satpy/etc/readers/hrit_msg.yaml @@ -68,11 +68,11 @@ file_types: HRIT_PRO: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGPrologueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-_________-PRO______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-_________-PRO______-{start_time:%Y%m%d%H%M}-__'] HRIT_EPI: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGEpilogueFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-_________-EPI______-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-_________-EPI______-{start_time:%Y%m%d%H%M}-__'] datasets: HRV: diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 8ae015e55d..6953292853 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -25,7 +25,10 @@ """ import logging -from datetime import datetime, timedelta +from datetime import timedelta +from tempfile import gettempdir +import os +from six import StringIO import numpy as np import xarray as xr @@ -85,12 +88,61 @@ } +def decompress(infile, outdir='.'): + """Will decompress an XRIT data file and return the path to the + decompressed file. It expect to find Eumetsat's xRITDecompress through the + environment variable XRIT_DECOMPRESS_PATH + """ + from subprocess import Popen, PIPE + cmd = os.environ.get('XRIT_DECOMPRESS_PATH', None) + if not cmd: + raise IOError("XRIT_DECOMPRESS_PATH is not defined" + + " (complete path to xRITDecompress)") + + infile = os.path.abspath(infile) + cwd = os.getcwd() + os.chdir(outdir) + + question = ("Did you set the environment variable " + + "XRIT_DECOMPRESS_PATH correctly?") + if not os.path.exists(cmd): + raise IOError(str(cmd) + " does not exist!\n" + question) + elif os.path.isdir(cmd): + raise IOError(str(cmd) + " is a directory!\n" + question) + + p = Popen([cmd, infile], stdout=PIPE) + stdout = StringIO(p.communicate()[0]) + status = p.returncode + os.chdir(cwd) + + outfile = '' + for line in stdout: + try: + k, v = [x.strip() for x in line.split(':', 1)] + except ValueError: + break + if k == 'Decompressed file': + outfile = v + break + + if status != 0: + raise IOError("xrit_decompress '%s', failed, status=%d" % (infile, status)) + if not outfile: + raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile) + return os.path.join(outdir, outfile) + + class HRITFileHandler(BaseFileHandler): """HRIT standard format reader.""" def __init__(self, filename, filename_info, filetype_info, hdr_info): """Initialize the reader.""" + if filename_info.get('compressed') == 'C': + logger.debug('Unpacking %s', filename) + filename = decompress(filename, gettempdir()) + filename_info['compressed'] = '' + super(HRITFileHandler, self).__init__(filename, filename_info, filetype_info) From 0ed472af012b763540cb1fe2c0904cca1d2bbe6b Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 1 Oct 2018 20:14:20 +0200 Subject: [PATCH 02/13] Split compressed and uncompressed xRIT filetypes --- satpy/etc/readers/hrit_electrol.yaml | 90 +++++++++++++++++----- satpy/etc/readers/hrit_goes.yaml | 36 +++++++-- satpy/etc/readers/hrit_msg.yaml | 108 +++++++++++++++++++++------ satpy/readers/hrit_base.py | 13 ++-- 4 files changed, 188 insertions(+), 59 deletions(-) diff --git a/satpy/etc/readers/hrit_electrol.yaml b/satpy/etc/readers/hrit_electrol.yaml index 7593273cfa..19ac8a14f2 100644 --- a/satpy/etc/readers/hrit_electrol.yaml +++ b/satpy/etc/readers/hrit_electrol.yaml @@ -8,52 +8,102 @@ reader: file_types: HRIT_00_6_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_00_7_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_00_9_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_03_8_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-03_8_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-03_8_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_06_4_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-06_4_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-06_4_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_08_0_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-08_0_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-08_0_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_08_7_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-08_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-08_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_09_7_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-09_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-09_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_10_7_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_11_9_4: file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-11_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-11_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_00_6_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_00_7_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_00_9_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-00_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_03_8_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-03_8_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_06_4_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-06_4_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_08_0_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-08_0_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_08_7_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-08_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_09_7_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-09_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_10_7_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_4, HRIT_EPI_4] + + HRIT_11_9_4_C: + file_reader: !!python/name:satpy.readers.hrit_electrol.HRITGOMSFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}4_____-11_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] requires: [HRIT_PRO_4, HRIT_EPI_4] HRIT_PRO_4: @@ -79,7 +129,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_00_6_4 + file_type: [HRIT_00_6_4, HRIT_00_6_4_C] '00_7': name: '00_7' @@ -95,7 +145,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_00_7_4 + file_type: [HRIT_00_7_4, HRIT_00_7_4_C] '00_9': name: '00_9' @@ -111,7 +161,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_00_9_4 + file_type: [HRIT_00_9_4, HRIT_00_9_4_C] '03_8': name: '03_8' @@ -127,7 +177,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_03_8_4 + file_type: [HRIT_03_8_4, HRIT_03_8_4_C] '06_4': name: '06_4' @@ -143,7 +193,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_06_4_4 + file_type: [HRIT_06_4_4, HRIT_06_4_4_C] '08_0': name: '08_0' @@ -159,7 +209,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_08_0_4 + file_type: [HRIT_08_0_4, HRIT_08_0_4_C] '08_7': name: '08_7' @@ -175,7 +225,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_08_7_4 + file_type: [HRIT_08_7_4, HRIT_08_7_4_C] '09_7': name: '09_7' @@ -191,7 +241,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_09_7_4 + file_type: [HRIT_09_7_4, HRIT_09_7_4_C] '10_7': name: '10_7' @@ -207,7 +257,7 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_10_7_4 + file_type: [HRIT_10_7_4, HRIT_10_7_4_C] '11_9': name: '11_9' @@ -223,4 +273,4 @@ datasets: counts: standard_name: counts units: '1' - file_type: HRIT_11_9_4 + file_type: [HRIT_11_9_4, HRIT_11_9_4_C] diff --git a/satpy/etc/readers/hrit_goes.yaml b/satpy/etc/readers/hrit_goes.yaml index 422e96ddfc..1ff5776463 100644 --- a/satpy/etc/readers/hrit_goes.yaml +++ b/satpy/etc/readers/hrit_goes.yaml @@ -12,22 +12,42 @@ reader: file_types: HRIT_00_7: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO_00] + HRIT_00_7_C: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-00_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO_00] + HRIT_03_9: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] + requires: [HRIT_PRO_03] + + HRIT_03_9_C: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-03_9_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] requires: [HRIT_PRO_03] HRIT_06_6: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] + requires: [HRIT_PRO_06] + + HRIT_06_6_C: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-06_6_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] requires: [HRIT_PRO_06] HRIT_10_7: file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] + requires: [HRIT_PRO_10] + + HRIT_10_7_C: + file_reader: !!python/name:satpy.readers.hrit_goes.HRITGOESFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:_<6s}______-10_7_{sublon:4s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] requires: [HRIT_PRO_10] HRIT_PRO_00: @@ -60,7 +80,7 @@ datasets: # units: W m-2 um-1 sr-1 counts: standard_name: counts - file_type: HRIT_00_7 + file_type: [HRIT_00_7, HRIT_00_7_C] '03_9': name: '03_9' @@ -75,7 +95,7 @@ datasets: # units: W m-2 um-1 sr-1 counts: standard_name: counts - file_type: HRIT_03_9 + file_type: [HRIT_03_9, HRIT_03_9_C] '06_6': name: '06_6' @@ -90,7 +110,7 @@ datasets: # units: W m-2 um-1 sr-1 counts: standard_name: counts - file_type: HRIT_06_6 + file_type: [HRIT_06_6, HRIT_06_6_C] '10_7': name: '10_7' @@ -105,4 +125,4 @@ datasets: # units: W m-2 um-1 sr-1 counts: standard_name: counts - file_type: HRIT_10_7 + file_type: [HRIT_10_7, HRIT_10_7_C] diff --git a/satpy/etc/readers/hrit_msg.yaml b/satpy/etc/readers/hrit_msg.yaml index e3770603be..619b1d11b4 100644 --- a/satpy/etc/readers/hrit_msg.yaml +++ b/satpy/etc/readers/hrit_msg.yaml @@ -8,62 +8,122 @@ reader: file_types: HRIT_HRV: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-HRV______-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-HRV______-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_016: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_016___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_016___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_039: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_039___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_039___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_087: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_087___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_087___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_097: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_097___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_097___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_108: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_108___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_108___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_120: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_120___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_120___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_IR_134: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_134___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_134___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_VIS006: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-VIS006___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-VIS006___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_VIS008: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-VIS008___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-VIS008___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_WV_062: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-WV_062___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-WV_062___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] requires: [HRIT_PRO, HRIT_EPI] HRIT_WV_073: file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler - file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-WV_073___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compressed:_<2s}'] + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-WV_073___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_HRV_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-HRV______-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_016_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_016___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_039_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_039___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_087_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_087___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_097_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_097___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_108_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_108___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_120_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_120___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_IR_134_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-IR_134___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_VIS006_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-VIS006___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_VIS008_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-VIS008___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_WV_062_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-WV_062___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] + requires: [HRIT_PRO, HRIT_EPI] + + HRIT_WV_073_C: + file_reader: !!python/name:satpy.readers.hrit_msg.HRITMSGFileHandler + file_patterns: ['{rate:1s}-000-{hrit_format:_<6s}-{platform_shortname:4s}_{service:_<7s}-WV_073___-{segment:_<9s}-{start_time:%Y%m%d%H%M}-C_'] requires: [HRIT_PRO, HRIT_EPI] HRIT_PRO: @@ -89,7 +149,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_HRV + file_type: [HRIT_HRV, HRIT_HRV_C] IR_016: name: IR_016 @@ -105,7 +165,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_016 + file_type: [HRIT_IR_016, HRIT_IR_016_C] IR_039: name: IR_039 @@ -121,7 +181,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_039 + file_type: [HRIT_IR_039, HRIT_IR_039_C] IR_087: name: IR_087 @@ -137,7 +197,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_087 + file_type: [HRIT_IR_087, HRIT_IR_087_C] IR_097: name: IR_097 @@ -153,7 +213,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_097 + file_type: [HRIT_IR_097, HRIT_IR_097_C] IR_108: name: IR_108 @@ -169,7 +229,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_108 + file_type: [HRIT_IR_108, HRIT_IR_108_C] IR_120: name: IR_120 @@ -185,7 +245,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_120 + file_type: [HRIT_IR_120, HRIT_IR_120_C] IR_134: name: IR_134 @@ -201,7 +261,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_IR_134 + file_type: [HRIT_IR_134, HRIT_IR_134_C] VIS006: name: VIS006 @@ -217,7 +277,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_VIS006 + file_type: [HRIT_VIS006, HRIT_VIS006_C] VIS008: name: VIS008 @@ -233,7 +293,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_VIS008 + file_type: [HRIT_VIS008, HRIT_VIS008_C] WV_062: name: WV_062 @@ -249,7 +309,7 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_WV_062 + file_type: [HRIT_WV_062, HRIT_WV_062_C] WV_073: name: WV_073 @@ -265,4 +325,4 @@ datasets: counts: standard_name: counts units: count - file_type: HRIT_WV_073 + file_type: [HRIT_WV_073, HRIT_WV_073_C] diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 6953292853..245c1ccb85 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -138,18 +138,17 @@ class HRITFileHandler(BaseFileHandler): def __init__(self, filename, filename_info, filetype_info, hdr_info): """Initialize the reader.""" - if filename_info.get('compressed') == 'C': - logger.debug('Unpacking %s', filename) - filename = decompress(filename, gettempdir()) - filename_info['compressed'] = '' - super(HRITFileHandler, self).__init__(filename, filename_info, filetype_info) - self.mda = {} - self._get_hd(hdr_info) + if self.mda.get('compression_flag_for_data'): + logger.debug('Unpacking %s', filename) + self.filename = decompress(filename, gettempdir()) + self.mda = {} + self._get_hd(hdr_info) + self._start_time = filename_info['start_time'] self._end_time = self._start_time + timedelta(minutes=15) From b3984e18fd80a78f74de4324239d84e441930bfa Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 1 Oct 2018 21:35:43 +0200 Subject: [PATCH 03/13] Log a warning when xrit decompression tool path is not found --- satpy/readers/hrit_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 245c1ccb85..a8c1f4b8b3 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -145,7 +145,10 @@ def __init__(self, filename, filename_info, filetype_info, hdr_info): if self.mda.get('compression_flag_for_data'): logger.debug('Unpacking %s', filename) - self.filename = decompress(filename, gettempdir()) + try: + self.filename = decompress(filename, gettempdir()) + except IOError as err: + logger.warning("Unpacking failed: %s", str(err)) self.mda = {} self._get_hd(hdr_info) From d713b4047bee06ab1bee13debb2fe427b2943ec8 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 2 Oct 2018 20:21:31 +0200 Subject: [PATCH 04/13] Use BytesIO for xrit decompression with python 3 --- satpy/readers/hrit_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index a8c1f4b8b3..5f4c4c25b8 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -28,7 +28,7 @@ from datetime import timedelta from tempfile import gettempdir import os -from six import StringIO +from six import BytesIO import numpy as np import xarray as xr @@ -111,7 +111,7 @@ def decompress(infile, outdir='.'): raise IOError(str(cmd) + " is a directory!\n" + question) p = Popen([cmd, infile], stdout=PIPE) - stdout = StringIO(p.communicate()[0]) + stdout = BytesIO(p.communicate()[0]) status = p.returncode os.chdir(cwd) From f56b6254095346e6b2135f29f0209002f462afea Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 2 Oct 2018 20:33:03 +0200 Subject: [PATCH 05/13] Fix more python 3 compatibility for xrit decompression --- satpy/readers/hrit_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 5f4c4c25b8..1c31577a02 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -115,13 +115,13 @@ def decompress(infile, outdir='.'): status = p.returncode os.chdir(cwd) - outfile = '' + outfile = b'' for line in stdout: try: - k, v = [x.strip() for x in line.split(':', 1)] + k, v = [x.strip() for x in line.split(b':', 1)] except ValueError: break - if k == 'Decompressed file': + if k == b'Decompressed file': outfile = v break @@ -129,7 +129,7 @@ def decompress(infile, outdir='.'): raise IOError("xrit_decompress '%s', failed, status=%d" % (infile, status)) if not outfile: raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile) - return os.path.join(outdir, outfile) + return os.path.join(outdir, outfile.decode('utf-8')) class HRITFileHandler(BaseFileHandler): From b9c14eece04f767e407b2202cbf162d1100148e1 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 21 Nov 2018 23:38:26 +0100 Subject: [PATCH 06/13] Split the xrit decompress function for maintainability --- satpy/readers/hrit_base.py | 47 +++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 1c31577a02..e7ab3de69a 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -88,21 +88,13 @@ } -def decompress(infile, outdir='.'): - """Will decompress an XRIT data file and return the path to the - decompressed file. It expect to find Eumetsat's xRITDecompress through the - environment variable XRIT_DECOMPRESS_PATH - """ - from subprocess import Popen, PIPE +def get_xritdecompress_cmd(): + """Find a valid binary for the xRITDecompress command.""" cmd = os.environ.get('XRIT_DECOMPRESS_PATH', None) if not cmd: raise IOError("XRIT_DECOMPRESS_PATH is not defined" + " (complete path to xRITDecompress)") - infile = os.path.abspath(infile) - cwd = os.getcwd() - os.chdir(outdir) - question = ("Did you set the environment variable " + "XRIT_DECOMPRESS_PATH correctly?") if not os.path.exists(cmd): @@ -110,11 +102,11 @@ def decompress(infile, outdir='.'): elif os.path.isdir(cmd): raise IOError(str(cmd) + " is a directory!\n" + question) - p = Popen([cmd, infile], stdout=PIPE) - stdout = BytesIO(p.communicate()[0]) - status = p.returncode - os.chdir(cwd) + return cmd + +def get_xritdecompress_output(stdout, status): + """Analyse the output of the xRITDecompress command call and return the file.""" outfile = b'' for line in stdout: try: @@ -125,10 +117,33 @@ def decompress(infile, outdir='.'): outfile = v break - if status != 0: - raise IOError("xrit_decompress '%s', failed, status=%d" % (infile, status)) if not outfile: raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile) + return outfile + +def decompress(infile, outdir='.'): + """Decompress an XRIT data file and return the path to the decompressed file. + + It expect to find Eumetsat's xRITDecompress through the environment variable + XRIT_DECOMPRESS_PATH. + """ + from subprocess import Popen, PIPE + + cmd = get_xritdecompress_cmd() + infile = os.path.abspath(infile) + cwd = os.getcwd() + os.chdir(outdir) + + p = Popen([cmd, infile], stdout=PIPE) + stdout = BytesIO(p.communicate()[0]) + status = p.returncode + os.chdir(cwd) + + if status != 0: + raise IOError("xrit_decompress '%s', failed, status=%d" % (infile, status)) + + outfile = get_xritdecompress_output(stdout, status) + return os.path.join(outdir, outfile.decode('utf-8')) From dc5e16273a0ec3440ff9be6aec7782b5d96c5c11 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 21 Nov 2018 23:39:05 +0100 Subject: [PATCH 07/13] Improve documentation for the hrit_base module --- satpy/readers/hrit_base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index e7ab3de69a..866ae5ac0a 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -20,7 +20,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""HRIT format reader +"""HRIT/LRIT format reader + +This module is the base module for all HRIT-based formats. Here, you will find +the common building blocks for hrit reading. + +One of the features here is the on-the-fly decompression of hrit files. It needs +a path to the xRITDecompress binary to be provided through the environment +variable called XRIT_DECOMPRESS_PATH. When compressed hrit files are then +encountered (files finishing with `.C_`), they are decompressed to the system's +temporary directory for reading. """ From b664ab49c64769563b496daa5a8a4196c387cedd Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 21 Nov 2018 23:41:36 +0100 Subject: [PATCH 08/13] Fix style --- satpy/readers/hrit_base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 866ae5ac0a..9d6e0c624f 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -101,11 +101,9 @@ def get_xritdecompress_cmd(): """Find a valid binary for the xRITDecompress command.""" cmd = os.environ.get('XRIT_DECOMPRESS_PATH', None) if not cmd: - raise IOError("XRIT_DECOMPRESS_PATH is not defined" + - " (complete path to xRITDecompress)") + raise IOError("XRIT_DECOMPRESS_PATH is not defined (complete path to xRITDecompress)") - question = ("Did you set the environment variable " + - "XRIT_DECOMPRESS_PATH correctly?") + question = ("Did you set the environment variable XRIT_DECOMPRESS_PATH correctly?") if not os.path.exists(cmd): raise IOError(str(cmd) + " does not exist!\n" + question) elif os.path.isdir(cmd): @@ -126,10 +124,9 @@ def get_xritdecompress_output(stdout, status): outfile = v break - if not outfile: - raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile) return outfile + def decompress(infile, outdir='.'): """Decompress an XRIT data file and return the path to the decompressed file. @@ -153,6 +150,9 @@ def decompress(infile, outdir='.'): outfile = get_xritdecompress_output(stdout, status) + if not outfile: + raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile) + return os.path.join(outdir, outfile.decode('utf-8')) From 51f0f8cf4c4958cac44df4d0a31966eeb857b9fa Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 22 Nov 2018 00:05:18 +0100 Subject: [PATCH 09/13] Add tests for hrit decompress functionality --- satpy/readers/hrit_base.py | 4 +-- satpy/tests/reader_tests/test_hrit_base.py | 36 +++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 9d6e0c624f..c9a268cd2c 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -112,7 +112,7 @@ def get_xritdecompress_cmd(): return cmd -def get_xritdecompress_output(stdout, status): +def get_xritdecompress_outfile(stdout): """Analyse the output of the xRITDecompress command call and return the file.""" outfile = b'' for line in stdout: @@ -148,7 +148,7 @@ def decompress(infile, outdir='.'): if status != 0: raise IOError("xrit_decompress '%s', failed, status=%d" % (infile, status)) - outfile = get_xritdecompress_output(stdout, status) + outfile = get_xritdecompress_outfile(stdout) if not outfile: raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile) diff --git a/satpy/tests/reader_tests/test_hrit_base.py b/satpy/tests/reader_tests/test_hrit_base.py index d0335724ae..614a20c9ab 100644 --- a/satpy/tests/reader_tests/test_hrit_base.py +++ b/satpy/tests/reader_tests/test_hrit_base.py @@ -26,7 +26,7 @@ import numpy as np -from satpy.readers.hrit_base import HRITFileHandler +from satpy.readers.hrit_base import HRITFileHandler, get_xritdecompress_cmd, get_xritdecompress_outfile if sys.version_info < (2, 7): import unittest2 as unittest @@ -39,6 +39,38 @@ import mock +class TestHRITDecompress(unittest.TestCase): + """Test the on-the-fly decompression.""" + + def test_xrit_cmd(self): + import os + from tempfile import gettempdir, mkstemp + old_env = os.environ.get('XRIT_DECOMPRESS_PATH', None) + + os.environ['XRIT_DECOMPRESS_PATH'] = '/path/to/my/bin' + self.assertRaises(IOError, get_xritdecompress_cmd) + + os.environ['XRIT_DECOMPRESS_PATH'] = gettempdir() + self.assertRaises(IOError, get_xritdecompress_cmd) + + handle, fname = mkstemp() + os.close(handle) + os.environ['XRIT_DECOMPRESS_PATH'] = fname + self.assertEqual(fname, get_xritdecompress_cmd()) + os.remove(fname) + + if old_env is not None: + os.environ['XRIT_DECOMPRESS_PATH'] = old_env + + def test_xrit_outfile(self): + stdout = ["Decompressed file: bla.__\n"] + outfile = get_xritdecompress_outfile(stdout) + self.assertEqual(outfile, 'bla.__') + + def test_decompress(self): + pass + + class TestHRITFileHandler(unittest.TestCase): """Test the HRITFileHandler.""" @@ -120,6 +152,8 @@ def suite(): loader = unittest.TestLoader() mysuite = unittest.TestSuite() mysuite.addTest(loader.loadTestsFromTestCase(TestHRITFileHandler)) + mysuite.addTest(loader.loadTestsFromTestCase(TestHRITDecompress)) + return mysuite From a6232e379e8f5680958f2c9bcb62504242d274d1 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 22 Nov 2018 11:05:46 +0100 Subject: [PATCH 10/13] Test xrit decompression functions --- satpy/readers/hrit_base.py | 3 +- satpy/tests/reader_tests/test_hrit_base.py | 42 +++++++++++++++------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index c9a268cd2c..d16fc2241e 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -38,6 +38,7 @@ from tempfile import gettempdir import os from six import BytesIO +from subprocess import Popen, PIPE import numpy as np import xarray as xr @@ -133,8 +134,6 @@ def decompress(infile, outdir='.'): It expect to find Eumetsat's xRITDecompress through the environment variable XRIT_DECOMPRESS_PATH. """ - from subprocess import Popen, PIPE - cmd = get_xritdecompress_cmd() infile = os.path.abspath(infile) cwd = os.getcwd() diff --git a/satpy/tests/reader_tests/test_hrit_base.py b/satpy/tests/reader_tests/test_hrit_base.py index 614a20c9ab..34745792a5 100644 --- a/satpy/tests/reader_tests/test_hrit_base.py +++ b/satpy/tests/reader_tests/test_hrit_base.py @@ -23,10 +23,12 @@ import sys from datetime import datetime +import os +from tempfile import gettempdir, NamedTemporaryFile import numpy as np -from satpy.readers.hrit_base import HRITFileHandler, get_xritdecompress_cmd, get_xritdecompress_outfile +from satpy.readers.hrit_base import HRITFileHandler, get_xritdecompress_cmd, get_xritdecompress_outfile, decompress if sys.version_info < (2, 7): import unittest2 as unittest @@ -43,8 +45,6 @@ class TestHRITDecompress(unittest.TestCase): """Test the on-the-fly decompression.""" def test_xrit_cmd(self): - import os - from tempfile import gettempdir, mkstemp old_env = os.environ.get('XRIT_DECOMPRESS_PATH', None) os.environ['XRIT_DECOMPRESS_PATH'] = '/path/to/my/bin' @@ -53,23 +53,41 @@ def test_xrit_cmd(self): os.environ['XRIT_DECOMPRESS_PATH'] = gettempdir() self.assertRaises(IOError, get_xritdecompress_cmd) - handle, fname = mkstemp() - os.close(handle) - os.environ['XRIT_DECOMPRESS_PATH'] = fname - self.assertEqual(fname, get_xritdecompress_cmd()) - os.remove(fname) + with NamedTemporaryFile() as fd: + os.environ['XRIT_DECOMPRESS_PATH'] = fd.name + fname = fd.name + res = get_xritdecompress_cmd() if old_env is not None: os.environ['XRIT_DECOMPRESS_PATH'] = old_env + else: + os.environ.pop('XRIT_DECOMPRESS_PATH') + + self.assertEqual(fname, res) def test_xrit_outfile(self): - stdout = ["Decompressed file: bla.__\n"] + stdout = [b"Decompressed file: bla.__\n"] outfile = get_xritdecompress_outfile(stdout) - self.assertEqual(outfile, 'bla.__') + self.assertEqual(outfile, b'bla.__') + + @mock.patch('satpy.readers.hrit_base.Popen') + def test_decompress(self, popen): + + popen.return_value.returncode = 0 + popen.return_value.communicate.return_value = [b"Decompressed file: bla.__\n"] + + old_env = os.environ.get('XRIT_DECOMPRESS_PATH', None) + + with NamedTemporaryFile() as fd: + os.environ['XRIT_DECOMPRESS_PATH'] = fd.name + res = decompress('bla.C_') - def test_decompress(self): - pass + if old_env is not None: + os.environ['XRIT_DECOMPRESS_PATH'] = old_env + else: + os.environ.pop('XRIT_DECOMPRESS_PATH') + self.assertEqual(res, './bla.__') class TestHRITFileHandler(unittest.TestCase): """Test the HRITFileHandler.""" From a6a0962ba009edb8c9693eae487f660c0e1e71f5 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 22 Nov 2018 11:15:45 +0100 Subject: [PATCH 11/13] Fix style --- satpy/tests/reader_tests/test_hrit_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/tests/reader_tests/test_hrit_base.py b/satpy/tests/reader_tests/test_hrit_base.py index 34745792a5..f5075e801c 100644 --- a/satpy/tests/reader_tests/test_hrit_base.py +++ b/satpy/tests/reader_tests/test_hrit_base.py @@ -89,6 +89,7 @@ def test_decompress(self, popen): self.assertEqual(res, './bla.__') + class TestHRITFileHandler(unittest.TestCase): """Test the HRITFileHandler.""" From d03dd73dc1ffa60b904b121e6d1b2a287b2057b7 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 22 Nov 2018 11:37:00 +0100 Subject: [PATCH 12/13] Add hrit readers info to the main documentation --- doc/source/readers.rst | 16 ++++++++++++++++ satpy/readers/hrit_base.py | 1 + satpy/readers/hrit_electrol.py | 3 ++- satpy/readers/hrit_goes.py | 3 ++- satpy/readers/hrit_jma.py | 3 ++- satpy/readers/hrit_msg.py | 6 ++---- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/doc/source/readers.rst b/doc/source/readers.rst index 9ee20f18d7..73cb663851 100644 --- a/doc/source/readers.rst +++ b/doc/source/readers.rst @@ -113,3 +113,19 @@ Adding a Reader to SatPy ======================== Coming soon... + +Implemented readers +=================== + +xRIT-based readers +------------------ + +.. automodule:: satpy.readers.hrit_base + +.. automodule:: satpy.readers.hrit_msg + +.. automodule:: satpy.readers.hrit_jma + +.. automodule:: satpy.readers.hrit_goes + +.. automodule:: satpy.readers.hrit_electrol diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index d16fc2241e..201f492829 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -21,6 +21,7 @@ # along with this program. If not, see . """HRIT/LRIT format reader +*************************** This module is the base module for all HRIT-based formats. Here, you will find the common building blocks for hrit reading. diff --git a/satpy/readers/hrit_electrol.py b/satpy/readers/hrit_electrol.py index 5eadd65104..dd5cb09835 100644 --- a/satpy/readers/hrit_electrol.py +++ b/satpy/readers/hrit_electrol.py @@ -22,7 +22,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""HRIT format reader. +"""HRIT format reader +********************** References: ELECTRO-L GROUND SEGMENT MSU-GS INSTRUMENT, diff --git a/satpy/readers/hrit_goes.py b/satpy/readers/hrit_goes.py index 75b99fd359..ce272c5e3c 100644 --- a/satpy/readers/hrit_goes.py +++ b/satpy/readers/hrit_goes.py @@ -21,7 +21,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""GOES HRIT format reader. +"""GOES HRIT format reader +**************************** References: LRIT/HRIT Mission Specific Implementation, February 2012 diff --git a/satpy/readers/hrit_jma.py b/satpy/readers/hrit_jma.py index c9f706557b..c8ed529b65 100644 --- a/satpy/readers/hrit_jma.py +++ b/satpy/readers/hrit_jma.py @@ -20,7 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""HRIT format reader for JMA data. +"""HRIT format reader for JMA data +************************************ References: JMA HRIT - Mission Specific Implementation diff --git a/satpy/readers/hrit_msg.py b/satpy/readers/hrit_msg.py index ab9ec7ece8..2352862e54 100644 --- a/satpy/readers/hrit_msg.py +++ b/satpy/readers/hrit_msg.py @@ -22,14 +22,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""SEVIRI HRIT format reader. +"""SEVIRI HRIT format reader +****************************** References: MSG Level 1.5 Image Data FormatDescription -TODO: -- HRV navigation - """ import logging From cf370c51d45af8026c5eea3ecc3ff85a6dc938f2 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 22 Nov 2018 14:08:10 +0100 Subject: [PATCH 13/13] Fix hrit decompression tests for windows --- satpy/tests/reader_tests/test_hrit_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_hrit_base.py b/satpy/tests/reader_tests/test_hrit_base.py index f5075e801c..6610dc21fa 100644 --- a/satpy/tests/reader_tests/test_hrit_base.py +++ b/satpy/tests/reader_tests/test_hrit_base.py @@ -87,7 +87,7 @@ def test_decompress(self, popen): else: os.environ.pop('XRIT_DECOMPRESS_PATH') - self.assertEqual(res, './bla.__') + self.assertEqual(res, os.path.join('.', 'bla.__')) class TestHRITFileHandler(unittest.TestCase):