From a3a871c7838ceef0158516509b871c2eb5f57a65 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 17 Oct 2024 15:03:06 +0200 Subject: [PATCH 1/5] fixing so milliseconds are used --- bin/vgac2pps.py | 2 +- level1c4pps/tests/test_vgac2pps.py | 14 +++++++++++--- level1c4pps/vgac2pps_lib.py | 9 +++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/bin/vgac2pps.py b/bin/vgac2pps.py index c42c076..c177c98 100644 --- a/bin/vgac2pps.py +++ b/bin/vgac2pps.py @@ -46,7 +46,7 @@ parser.add_argument('-all_ch', '--all_channels', action='store_true', help="Save all 21 channels to level1c4pps file.") parser.add_argument('-n19', '--as_noaa19', - options=[version for version in SBAF], + choices=[version for version in SBAF], default=None, help=("Save only the AVHRR (1,2, 3B, 4, 5) channels to level1c4pps file. " "And apply SBAFs to the channels.")) diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 37fc9c3..96d779c 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -80,9 +80,16 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, - 'units': 'Milliseconds since 1970-01-01', - 'complevel': 4, '_FillValue': -1.0}, + 'scanline_timestamps': {'dtype': 'float', + 'zlib': True, + 'units': 'milliseconds since 1970-01-01', + 'complevel': 4, + '_FillValue': -1.0}, + 'time': {'dtype': 'float', + 'zlib': True, + 'units': 'milliseconds since 1970-01-01', + 'complevel': 4, + '_FillValue': -1.0}, } encoding = vgac2pps.get_encoding_viirs(self.scene) self.assertDictEqual(encoding, encoding_exp) @@ -121,6 +128,7 @@ def test_process_one_scene(self): np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor, 1.0, decimal=4) + assert pps_nc.variables["scanline_timestamps"].units == 'milliseconds since 1970-01-01' def test_process_one_scene_n19(self): """Test process one scene for one example file.""" diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 2864538..17720c8 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -357,10 +357,11 @@ def convert_to_noaa19(scene, sbaf_version): def get_encoding_viirs(scene): """Get netcdf encoding for all datasets.""" - return get_encoding(scene, - BANDNAMES, - PPS_TAGNAMES, - chunks=None) + encoding = get_encoding(scene, BANDNAMES, PPS_TAGNAMES, chunks=None) + encoding["scanline_timestamps"]["dtype"] = "float" + encoding["scanline_timestamps"]["units"] = "milliseconds since 1970-01-01" + encoding["time"] = encoding["scanline_timestamps"] + return encoding def set_header_and_band_attrs(scene, orbit_n=0): From 14eaf7e2dadbd576c334183f0e609b6e61be4375 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Fri, 18 Oct 2024 07:38:50 +0200 Subject: [PATCH 2/5] adding the fix in common function --- level1c4pps/__init__.py | 8 +++++--- level1c4pps/tests/test_eumgacfdr2pps.py | 4 ++-- level1c4pps/tests/test_gac2pps.py | 4 ++-- level1c4pps/tests/test_vgac2pps.py | 7 +------ level1c4pps/vgac2pps_lib.py | 9 ++++----- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 50d276b..087830e 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -320,9 +320,11 @@ def get_band_encoding(dataset, bandnames, pps_tagnames, chunks=None): 'complevel': 4, '_FillValue': -32001.0} elif name in ['scanline_timestamps']: # pygac scanline_timestamps - enc = {'dtype': 'int64', 'zlib': True, - 'units': 'Milliseconds since 1970-01-01', - 'complevel': 4, '_FillValue': -1.0} + enc = {'dtype': 'float64', + 'zlib': True, + 'units': 'milliseconds since 1970-01-01', + 'complevel': 4, + '_FillValue': -1.0} if not enc: raise ValueError('Unsupported band: {}'.format(name)) return name, enc diff --git a/level1c4pps/tests/test_eumgacfdr2pps.py b/level1c4pps/tests/test_eumgacfdr2pps.py index 52784fa..cffb042 100644 --- a/level1c4pps/tests/test_eumgacfdr2pps.py +++ b/level1c4pps/tests/test_eumgacfdr2pps.py @@ -81,8 +81,8 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, - 'units': 'Milliseconds since 1970-01-01', + 'scanline_timestamps': {'dtype': 'float64', 'zlib': True, + 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, } encoding = eumgacfdr2pps.get_encoding_gac(self.scene) diff --git a/level1c4pps/tests/test_gac2pps.py b/level1c4pps/tests/test_gac2pps.py index 2df66b7..e9419f0 100644 --- a/level1c4pps/tests/test_gac2pps.py +++ b/level1c4pps/tests/test_gac2pps.py @@ -80,8 +80,8 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, - 'units': 'Milliseconds since 1970-01-01', + 'scanline_timestamps': {'dtype': 'float64', 'zlib': True, + 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, } encoding = gac2pps.get_encoding_gac(self.scene) diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 96d779c..3a9baec 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -80,16 +80,11 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'float', + 'scanline_timestamps': {'dtype': 'float64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, - 'time': {'dtype': 'float', - 'zlib': True, - 'units': 'milliseconds since 1970-01-01', - 'complevel': 4, - '_FillValue': -1.0}, } encoding = vgac2pps.get_encoding_viirs(self.scene) self.assertDictEqual(encoding, encoding_exp) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 17720c8..2864538 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -357,11 +357,10 @@ def convert_to_noaa19(scene, sbaf_version): def get_encoding_viirs(scene): """Get netcdf encoding for all datasets.""" - encoding = get_encoding(scene, BANDNAMES, PPS_TAGNAMES, chunks=None) - encoding["scanline_timestamps"]["dtype"] = "float" - encoding["scanline_timestamps"]["units"] = "milliseconds since 1970-01-01" - encoding["time"] = encoding["scanline_timestamps"] - return encoding + return get_encoding(scene, + BANDNAMES, + PPS_TAGNAMES, + chunks=None) def set_header_and_band_attrs(scene, orbit_n=0): From 24698d39cd229426cfafd7d8d82780461b5d2553 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 21 Nov 2024 14:12:41 +0100 Subject: [PATCH 3/5] casting to datetime64/ms --- level1c4pps/__init__.py | 2 +- level1c4pps/tests/test_vgac2pps.py | 2 +- level1c4pps/vgac2pps_lib.py | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 087830e..94fd99e 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -320,7 +320,7 @@ def get_band_encoding(dataset, bandnames, pps_tagnames, chunks=None): 'complevel': 4, '_FillValue': -32001.0} elif name in ['scanline_timestamps']: # pygac scanline_timestamps - enc = {'dtype': 'float64', + enc = {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 3a9baec..2f16367 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -80,7 +80,7 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'float64', + 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 2864538..dcc130a 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -456,6 +456,13 @@ def split_scene_at_midnight(scene): return [scene] +def fix_timestamp_datatype(scene, encoding): + """Fix time datatype.""" + param = "scanline_timestamps" + if "milliseconds" in encoding[param]["units"]: + scene[param].data = scene[param].data.astype('datetime64[ms]') + + def process_one_scene(scene_files, out_path, engine="h5netcdf", all_channels=False, pps_channels=False, orbit_n=0, noaa19_sbaf_version=None, avhrr_channels=False, @@ -506,6 +513,7 @@ def process_one_scene(scene_files, out_path, engine="h5netcdf", filename = compose_filename(scn_, out_path, instrument=sensor, band=irch) encoding = get_encoding_viirs(scn_) + fix_timestamp_datatype(scn_, encoding) scn_.save_datasets(writer="cf", filename=filename, From de72bd1968f56af2e0dfa88b5b251bc98878823a Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 21 Nov 2024 14:21:01 +0100 Subject: [PATCH 4/5] adding test --- level1c4pps/tests/test_vgac2pps.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 2f16367..aa9ecfd 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -123,7 +123,10 @@ def test_process_one_scene(self): np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor, 1.0, decimal=4) - assert pps_nc.variables["scanline_timestamps"].units == 'milliseconds since 1970-01-01' + assert ( + pps_nc.variables["scanline_timestamps"].units == "milliseconds since 1970-01-01" + and pps_nc.variables["scanline_timestamps"].dtype == "int" + ) def test_process_one_scene_n19(self): """Test process one scene for one example file.""" From f060e4886dd28d2919a1c5f3d367b0c8ddc85f6a Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 21 Nov 2024 14:38:34 +0100 Subject: [PATCH 5/5] fixing tests --- level1c4pps/tests/test_eumgacfdr2pps.py | 2 +- level1c4pps/tests/test_gac2pps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/level1c4pps/tests/test_eumgacfdr2pps.py b/level1c4pps/tests/test_eumgacfdr2pps.py index cffb042..fd519df 100644 --- a/level1c4pps/tests/test_eumgacfdr2pps.py +++ b/level1c4pps/tests/test_eumgacfdr2pps.py @@ -81,7 +81,7 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'float64', 'zlib': True, + 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, } diff --git a/level1c4pps/tests/test_gac2pps.py b/level1c4pps/tests/test_gac2pps.py index e9419f0..e2ed1eb 100644 --- a/level1c4pps/tests/test_gac2pps.py +++ b/level1c4pps/tests/test_gac2pps.py @@ -80,7 +80,7 @@ def test_get_encoding(self): 'add_offset': 273.15}, 'qual_flags': {'dtype': 'int16', 'zlib': True, 'complevel': 4, '_FillValue': -32001.0}, - 'scanline_timestamps': {'dtype': 'float64', 'zlib': True, + 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, }