try to work around some GPS issues - #141 #22

gopro_overlay/gpmf/calc.py

@@ -26,11 +26,15 @@ def __init__(self, cori_timestamp: Timeunit):
     def next_packet(self, timestamp, samples_before_this, num_samples):
 
         if self._first_timestamp is not None and self._last_timestamp is not None and timestamp < self._last_timestamp:
-            # This is definitely wrong - need all the SHUT timings from the joined files...
-            self._adjust += self._last_timestamp
-            log(f"Joined file detected... adjusting by {self._adjust}")
-            self._first_timestamp = timestamp
-            self._last_timestamp = timestamp
+            delta = abs(self._last_timestamp - timestamp)
+            if delta < timeunits(seconds=1):
+                log(f"GPS Time correction.. Step back {delta} - possible GPS issue?")
+            else:
+                # This is definitely wrong - need all the SHUT timings from the joined files...
+                self._adjust += self._last_timestamp
+                log(f"Joined file detected... adjusting by {self._adjust}")
+                self._first_timestamp = timestamp
+                self._last_timestamp = timestamp
 
         if self._first_timestamp is None:
             self._first_timestamp = timestamp

gopro_overlay/gpmf/gpmf.py

@@ -3,8 +3,9 @@
 import itertools
 import struct
 from enum import Enum
-from typing import List, TypeVar
+from typing import List, TypeVar, Optional
 
+from gopro_overlay.log import log
 from gopro_overlay.timeunits import timeunits
 
 T = TypeVar("T")
@@ -104,13 +105,15 @@ def _interpret_string(item, **kwargs):
     return str(item.rawdata, encoding='unicode_escape', errors="replace").strip('\0')
 
 
-def _interpret_gps_timestamp(item, **kwargs):
-    return datetime.datetime.strptime(
-        item.rawdata.decode(
-            'utf-8',
-            errors='replace'
-        ),
-        '%y%m%d%H%M%S.%f').replace(tzinfo=datetime.timezone.utc)
+def _interpret_gps_timestamp(item, **kwargs) -> Optional[datetime.datetime]:
+    date_string = item.rawdata.decode('utf-8', errors='replace')
+    try:
+        return datetime.datetime.strptime(
+            date_string,
+            '%y%m%d%H%M%S.%f').replace(tzinfo=datetime.timezone.utc)
+    except ValueError:
+        log(f"Couldn't parse GPS timestamp -: {date_string} - Known GoPro GPS firmware issue :- "
+            f"https://github.com/gopro/gpmf-parser/issues/162")
 
 
 def _struct_mapping_for(item, repeat=None):

gopro_overlay/gpmf/visitors/gps.py

@@ -6,14 +6,15 @@
 from gopro_overlay.gpmf.calc import PacketTimeCalculator
 from gopro_overlay.gpmd_filters import NullGPSLockFilter, GPSLockComponents
 from gopro_overlay.gpmf import GPSFix, GPS5, interpret_item, GPS9, GPS_FIXED
+from gopro_overlay.log import log
 from gopro_overlay.point import Point
 from gopro_overlay.timeunits import Timeunit
 
 
 @dataclasses.dataclass(frozen=True)
 class GPS5Components:
     samples: int
-    timestamp: int
+    timestamp: Timeunit
     basetime: datetime.datetime
     fix: GPSFix
     dop: float
@@ -29,17 +30,27 @@ def __init__(self, units, calculator: PacketTimeCalculator, on_item=lambda c, e:
         self._total_samples = 0
         self._frame_calculator = calculator
         self._tracker = gps_lock_filter
+        self._short_packet_count = 0
 
     def convert(self, counter, components: GPS5Components):
 
         # Turns out GPS5 can contain no points. Possibly accompanied by EMPT packet?
-        if len(components.points) == 0:
+        point_count = len(components.points)
+
+        if point_count == 0:
             return
 
+        # There should be 18 points per packet generally..
+        if point_count <= 10:
+            self._short_packet_count += 1
+            if self._short_packet_count % 10 == 0:
+                log(f"Have seen {self._short_packet_count} suspicious GPS packets. Last was {point_count}/18. GPS is misbehaving? - See https://github.com/time4tea/gopro-dashboard-overlay/issues/141")
+
+
         sample_time_calculator = self._frame_calculator.next_packet(
             components.timestamp,
             self._total_samples,
-            len(components.points)
+            point_count
         )
 
         gpsfix = components.fix
@@ -62,6 +73,7 @@ def convert(self, counter, components: GPS5Components):
                     timestamp=self._units.Quantity(sample_frame_timestamp.millis(), self._units.number),
                     dop=self._units.Quantity(components.dop, self._units.number),
                     packet=self._units.Quantity(counter, self._units.number),
+                    packet_count=self._units.Quantity(point_count),
                     packet_index=self._units.Quantity(index, self._units.number),
                     point=position,
                     speed=speed,
@@ -70,7 +82,7 @@ def convert(self, counter, components: GPS5Components):
                     gpslock=self._units.Quantity(calculated_fix.value),
                 )
             )
-        self._total_samples += len(components.points)
+        self._total_samples += point_count
 
 
 class GPS5StreamVisitor:
@@ -82,7 +94,7 @@ def __init__(self, on_end):
         self._fix: Optional[GPSFix] = None
         self._scale: Optional[int] = None
         self._points: Optional[List[GPS5]] = None
-        self._timestamp: Optional[int] = None
+        self._timestamp: Optional[Timeunit] = None
         self._dop: Optional[float] = None
 
     def vi_STMP(self, item):
@@ -107,15 +119,18 @@ def vi_GPS5(self, item):
         self._points = interpret_item(item, scale=self._scale)
 
     def v_end(self):
-        self._on_end(GPS5Components(
-            samples=self._samples,
-            timestamp=self._timestamp,
-            basetime=self._basetime,
-            fix=self._fix,
-            dop=self._dop,
-            scale=self._scale,
-            points=self._points
-        ))
+        if self._basetime is not None:
+            self._on_end(GPS5Components(
+                samples=self._samples,
+                timestamp=self._timestamp,
+                basetime=self._basetime,
+                fix=self._fix,
+                dop=self._dop,
+                scale=self._scale,
+                points=self._points
+            ))
+        else:
+            log(f"No GPS Date :- Skipping Record with {len(self._points)} samples")
 
 
 class GPS5Visitor: