Merge pull request #96 from BengtRydberg/vgac_time_units_fix

vgac time units
foua-pps · Nov 21, 2024 · deed643 · deed643
2 parents 0b7b95d + f060e48
commit deed643
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 8 deletions.
diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py
@@ -320,9 +320,11 @@ def get_band_encoding(dataset, bandnames, pps_tagnames, chunks=None):
                'complevel': 4, '_FillValue': -32001.0}
     elif name in ['scanline_timestamps']:
         # pygac scanline_timestamps
-        enc = {'dtype': 'int64', 'zlib': True,
-               'units': 'Milliseconds since 1970-01-01',
-               'complevel': 4, '_FillValue': -1.0}
+        enc = {'dtype': 'int64',
+               'zlib': True,
+               'units': 'milliseconds since 1970-01-01',
+               'complevel': 4,
+               '_FillValue': -1.0}
     if not enc:
         raise ValueError('Unsupported band: {}'.format(name))
     return name, enc

diff --git a/level1c4pps/tests/test_eumgacfdr2pps.py b/level1c4pps/tests/test_eumgacfdr2pps.py
@@ -82,7 +82,7 @@ def test_get_encoding(self):
             'qual_flags':  {'dtype': 'int16', 'zlib': True,
                             'complevel': 4, '_FillValue': -32001.0},
             'scanline_timestamps': {'dtype': 'int64', 'zlib': True,
-                                    'units': 'Milliseconds since 1970-01-01',
+                                    'units': 'milliseconds since 1970-01-01',
                                     'complevel': 4, '_FillValue': -1.0},
         }
         encoding = eumgacfdr2pps.get_encoding_gac(self.scene)

diff --git a/level1c4pps/tests/test_gac2pps.py b/level1c4pps/tests/test_gac2pps.py
@@ -81,7 +81,7 @@ def test_get_encoding(self):
             'qual_flags':  {'dtype': 'int16', 'zlib': True,
                             'complevel': 4, '_FillValue': -32001.0},
             'scanline_timestamps': {'dtype': 'int64', 'zlib': True,
-                                    'units': 'Milliseconds since 1970-01-01',
+                                    'units': 'milliseconds since 1970-01-01',
                                     'complevel': 4, '_FillValue': -1.0},
         }
         encoding = gac2pps.get_encoding_gac(self.scene)

diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py
@@ -80,9 +80,11 @@ def test_get_encoding(self):
                        'add_offset': 273.15},
             'qual_flags':  {'dtype': 'int16', 'zlib': True,
                             'complevel': 4, '_FillValue': -32001.0},
-            'scanline_timestamps': {'dtype': 'int64', 'zlib': True,
-                                    'units': 'Milliseconds since 1970-01-01',
-                                    'complevel': 4, '_FillValue': -1.0},
+            'scanline_timestamps': {'dtype': 'int64',
+                                    'zlib': True,
+                                    'units': 'milliseconds since 1970-01-01',
+                                    'complevel': 4,
+                                    '_FillValue': -1.0},
         }
         encoding = vgac2pps.get_encoding_viirs(self.scene)
         self.assertDictEqual(encoding, encoding_exp)
@@ -121,6 +123,10 @@ def test_process_one_scene(self):
 
         np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor,
                                        1.0, decimal=4)
+        assert (
+            pps_nc.variables["scanline_timestamps"].units == "milliseconds since 1970-01-01"
+            and pps_nc.variables["scanline_timestamps"].dtype == "int"
+        )
 
     def test_process_one_scene_n19(self):
         """Test process one scene for one example file."""

diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py
@@ -456,6 +456,13 @@ def split_scene_at_midnight(scene):
     return [scene]
 
 
+def fix_timestamp_datatype(scene, encoding):
+    """Fix time datatype."""
+    param = "scanline_timestamps"
+    if "milliseconds" in encoding[param]["units"]:
+        scene[param].data = scene[param].data.astype('datetime64[ms]')
+
+
 def process_one_scene(scene_files, out_path, engine="h5netcdf",
                       all_channels=False, pps_channels=False, orbit_n=0,
                       noaa19_sbaf_version=None, avhrr_channels=False,
@@ -506,6 +513,7 @@ def process_one_scene(scene_files, out_path, engine="h5netcdf",
 
         filename = compose_filename(scn_, out_path, instrument=sensor, band=irch)
         encoding = get_encoding_viirs(scn_)
+        fix_timestamp_datatype(scn_, encoding)
 
         scn_.save_datasets(writer="cf",
                            filename=filename,