fix: Fixed various issues relating to sorting during daylight saving …

…transitions (1 hour dev time)

custom_components/octopus_energy/api_client/__init__.py

@@ -521,7 +521,7 @@ def get_valid_from(rate):
   return rate["valid_from"]
 
 def get_start(rate):
-  return rate["start"]
+  return (rate["start"].timestamp(), rate["start"].fold)
 
 def rates_to_thirty_minute_increments(data, period_from: datetime, period_to: datetime, tariff_code: str, price_cap: float = None, favour_direct_debit_rates = True):
   """Process the collection of rates to ensure they're in 30 minute periods"""
@@ -856,7 +856,7 @@ async def async_get_greenness_forecast(self) -> list[GreennessForecast]:
                                                              item["greennessIndex"],
                                                              item["highlightFlag"]),
                           response_body["data"]["greennessForecast"]))
-          forecast.sort(key=lambda item: item.start)
+          forecast.sort(key=lambda item: (item.start.timestamp(), item.start.fold))
           return forecast
 
     except TimeoutError:
@@ -1606,7 +1606,7 @@ async def async_spin_wheel_of_fortune(self, account_id: str, is_electricity: boo
       raise TimeoutException()
 
   def __get_interval_end(self, item):
-    return item["end"]
+    return (item["end"].timestamp(), item["end"].fold)
 
   def __is_night_rate(self, rate, is_smart_meter):
     # Normally the economy seven night rate is between 12am and 7am UK time

custom_components/octopus_energy/coordinators/previous_consumption_and_rates.py

@@ -37,7 +37,7 @@
 _LOGGER = logging.getLogger(__name__)
 
 def __get_interval_end(item):
-    return item["end"]
+    return (item["end"].timestamp(), item["end"].fold)
 
 def __sort_consumption(consumption_data):
   sorted = consumption_data.copy()
@@ -179,8 +179,8 @@ async def async_enhance_with_historic_consumption(
   historic_weekday_consumptions = remove_old_consumptions(historic_weekday_consumptions, earliest_weekday_start)
   historic_weekend_consumptions = remove_old_consumptions(historic_weekend_consumptions, earliest_weekend_start)
 
-  historic_weekday_consumptions.sort(key=lambda x: x["start"])
-  historic_weekend_consumptions.sort(key=lambda x: x["start"])
+  historic_weekday_consumptions.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
+  historic_weekend_consumptions.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
   current_weekday_earliest_start = historic_weekday_consumptions[0]["start"] if historic_weekday_consumptions is not None and len(historic_weekday_consumptions) > 0 else None
   current_weekday_latest_end = historic_weekday_consumptions[-1]["end"] if historic_weekday_consumptions is not None and len(historic_weekday_consumptions) > 0 else None

custom_components/octopus_energy/electricity/__init__.py

@@ -1,13 +1,11 @@
-import datetime
 import logging
 
 from ..utils.conversions import value_inc_vat_to_pounds
-from ..utils import get_off_peak_cost
 
 _LOGGER = logging.getLogger(__name__)
 
 def __get_to(item):
-    return item["end"]
+    return (item["end"].timestamp(), item["end"].fold)
 
 def __sort_consumption(consumption_data):
   sorted = consumption_data.copy()

custom_components/octopus_energy/gas/__init__.py

@@ -4,7 +4,7 @@
 _LOGGER = logging.getLogger(__name__)
 
 def __get_to(item):
-    return item["end"]
+    return (item["end"].timestamp(), item["end"].fold)
 
 def __sort_consumption(consumption_data):
   sorted = consumption_data.copy()

custom_components/octopus_energy/target_rates/__init__.py

@@ -102,7 +102,7 @@ def get_rates(current_date: datetime, rates: list, target_hours: float):
   return applicable_rates
 
 def __get_valid_to(rate):
-  return rate["end"]
+  return (rate["end"].timestamp(), rate["end"].fold)
 
 def calculate_continuous_times(
     applicable_rates: list,
@@ -203,14 +203,14 @@ def calculate_intermittent_times(
 
   if find_last_rates:
     if search_for_highest_rate:
-      applicable_rates.sort(key= lambda rate: (-rate["value_inc_vat"], -rate["end"].timestamp()))
+      applicable_rates.sort(key= lambda rate: (-rate["value_inc_vat"], -rate["end"].timestamp(), -rate["end"].fold))
     else:
-      applicable_rates.sort(key= lambda rate: (rate["value_inc_vat"], -rate["end"].timestamp()))
+      applicable_rates.sort(key= lambda rate: (rate["value_inc_vat"], -rate["end"].timestamp(), -rate["end"].fold))
   else:
     if search_for_highest_rate:
-      applicable_rates.sort(key= lambda rate: (-rate["value_inc_vat"], rate["end"]))
+      applicable_rates.sort(key= lambda rate: (-rate["value_inc_vat"], rate["end"], rate["end"].fold))
     else:
-      applicable_rates.sort(key= lambda rate: (rate["value_inc_vat"], rate["end"]))
+      applicable_rates.sort(key= lambda rate: (rate["value_inc_vat"], rate["end"], rate["end"].fold))
 
   applicable_rates = list(filter(lambda rate: (min_rate is None or rate["value_inc_vat"] >= min_rate) and (max_rate is None or rate["value_inc_vat"] <= max_rate), applicable_rates))
 

custom_components/octopus_energy/utils/rate_information.py

@@ -72,7 +72,7 @@ def get_current_rate_information(rates, now: datetime):
   return None
 
 def get_from(rate):
-  return rate["start"]
+  return (rate["start"].timestamp(), rate["start"].fold)
 
 def get_previous_rate_information(rates, now: datetime):
   current_rate = None

tests/integration/__init__.py

@@ -61,7 +61,7 @@ def create_consumption_data(period_from: datetime, period_to: datetime, reverse
 
   if reverse == True:
     def get_interval_start(item):
-      return item["start"]
+      return (item["start"].timestamp(), item["start"].fold)
 
     consumption.sort(key=get_interval_start, reverse=True)
 

tests/unit/__init__.py

@@ -5,7 +5,7 @@
 
 logging.getLogger().setLevel(logging.DEBUG)
 
-def create_consumption_data(period_from, period_to, reverse = False, from_key = "start", to_key = "end"):
+def create_consumption_data(period_from: datetime, period_to: datetime, reverse = False, from_key = "start", to_key = "end"):
   consumption = []
   current_valid_from = period_from
   current_valid_to = None
@@ -22,13 +22,13 @@ def create_consumption_data(period_from, period_to, reverse = False, from_key =
 
   if reverse == True:
     def get_interval_start(item):
-      return item[from_key]
+      return (item[from_key].timestamp(), item[from_key].fold)
 
     consumption.sort(key=get_interval_start, reverse=True)
 
   return consumption
 
-def create_rate_data(period_from, period_to, expected_rates: list):
+def create_rate_data(period_from: datetime, period_to: datetime, expected_rates: list):
   rates = []
   current_valid_from = period_from
   current_valid_to = None

tests/unit/coordinators/test_async_enhance_with_historic_consumption.py

@@ -220,7 +220,7 @@ async def async_mocked_get_electricity_consumption(*args, **kwargs):
     assert result is not None
     assert result.historic_weekday_consumption is not None
     assert len(result.historic_weekday_consumption) == 48 * 15
-    expected_missing_weekday_times.sort(key=lambda x: x["start"])
+    expected_missing_weekday_times.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
     expected_weekday_times = expected_missing_weekday_times + [
       { "start": datetime.strptime("2024-09-16T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z"), "end": datetime.strptime("2024-09-17T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z") },
@@ -236,7 +236,7 @@ async def async_mocked_get_electricity_consumption(*args, **kwargs):
 
     assert result.historic_weekend_consumption is not None
     assert len(result.historic_weekend_consumption) == 48 * 8
-    expected_missing_weekend_times.sort(key=lambda x: x["start"])
+    expected_missing_weekend_times.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
     expected_weekend_times = expected_missing_weekend_times + [
       { "start": datetime.strptime("2024-09-15T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z"), "end": datetime.strptime("2024-09-16T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z") }

tests/unit/coordinators/test_async_refresh_electricity_rates_data.py

@@ -10,6 +10,7 @@
 from custom_components.octopus_energy.api_client.intelligent_dispatches import IntelligentDispatchItem, IntelligentDispatches
 from custom_components.octopus_energy.api_client.intelligent_device import IntelligentDevice
 from custom_components.octopus_energy.coordinators.intelligent_dispatches import IntelligentDispatchesCoordinatorResult
+from zoneinfo import ZoneInfo
 
 current = datetime.strptime("2023-07-14T10:30:01+01:00", "%Y-%m-%dT%H:%M:%S%z")
 period_from = datetime.strptime("2023-07-14T00:00:00+01:00", "%Y-%m-%dT%H:%M:%S%z")
@@ -38,7 +39,7 @@ def get_account_info(is_active_agreement = True, tariff_code = "E-1R-SUPER-GREEN
         "agreements": [
           {
             "start": "2023-07-01T00:00:00+01:00" if is_active_agreement else "2023-08-01T00:00:00+01:00",
-            "end": "2023-08-01T00:00:00+01:00" if is_active_agreement else "2023-09-01T00:00:00+01:00",
+            "end": "2025-08-01T00:00:00+01:00" if is_active_agreement else "2023-09-01T00:00:00+01:00",
             "tariff_code": tariff_code,
             "product_code": product_code
           }
@@ -217,7 +218,7 @@ async def test_when_existing_rates_is_none_then_rates_retrieved(existing_rates):
   expected_rates_unsorted = expected_rates.copy()
 
   # Put rates into reverse order to make sure sorting works
-  expected_rates.sort(key=lambda rate: rate["start"], reverse=True)
+  expected_rates.sort(key=lambda rate: (rate["start"].timestamp(), rate["start"].fold), reverse=True)
 
   assert expected_rates[-1]["start"] == expected_period_from
   assert expected_rates[0]["end"] == expected_period_to
@@ -287,7 +288,7 @@ async def test_when_dispatches_is_not_defined_and_existing_rates_is_none_then_ra
   expected_rates_unsorted = expected_rates.copy()
 
   # Put rates into reverse order to make sure sorting works
-  expected_rates.sort(key=lambda rate: rate["start"], reverse=True)
+  expected_rates.sort(key=lambda rate: (rate["start"].timestamp(), rate["start"].fold), reverse=True)
 
   assert expected_rates[-1]["start"] == expected_period_from
   assert expected_rates[0]["end"] == expected_period_to
@@ -959,7 +960,7 @@ async def test_when_rates_change_correctly_then_unique_rates_changed_event_fired
   expected_rates = create_rate_data(expected_period_from, expected_period_to, current_unique_rates)
 
   # Put rates into reverse order to make sure sorting works
-  expected_rates.sort(key=lambda rate: rate["start"], reverse=True)
+  expected_rates.sort(key=lambda rate: (rate["start"].timestamp(), rate["start"].fold), reverse=True)
 
   assert expected_rates[-1]["start"] == expected_period_from
   assert expected_rates[0]["end"] == expected_period_to
@@ -997,4 +998,97 @@ async def unique_rates_changed(name, metadata):
       unique_rates_changed=unique_rates_changed
     )
 
-  assert unique_rates_changed_called == expected_unique_rates_changed_event_fired
+  assert unique_rates_changed_called == expected_unique_rates_changed_event_fired
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("existing_rates",[
+  (None),
+  (ElectricityRatesCoordinatorResult(period_from, 1, [])),
+  (ElectricityRatesCoordinatorResult(period_from, 1, None)),
+])
+async def test_when_clocks_change_then_rates_are_correct(existing_rates):
+  current = datetime(2024, 10, 27, 10, 30, tzinfo=ZoneInfo(key='Europe/London'))
+  expected_period_from = datetime.strptime("2024-10-26T00:00:00+01:00", "%Y-%m-%dT%H:%M:%S%z")
+  expected_period_to = datetime.strptime("2024-10-29T00:00:00+00:00", "%Y-%m-%dT%H:%M:%S%z")
+  expected_rates = create_rate_data(datetime(2024, 10, 26, 0, 0, tzinfo=ZoneInfo(key='Europe/London')), datetime(2024, 10, 27, 1, 30, tzinfo=ZoneInfo(key='Europe/London')), [1])
+  expected_rates.append({
+    "start": datetime(2024, 10, 27, 1, 30, tzinfo=ZoneInfo(key='Europe/London')),
+    "end": datetime(2024, 10, 27, 1, 0, fold=1, tzinfo=ZoneInfo(key='Europe/London')),
+    "value_inc_vat": 1,
+    "tariff_code": "E-1R-Test-L",
+    "is_capped": False
+  })
+  expected_rates.append({
+    "start": datetime(2024, 10, 27, 1, 0, fold=1, tzinfo=ZoneInfo(key='Europe/London')),
+    "end": datetime(2024, 10, 27, 1, 30, fold=1, tzinfo=ZoneInfo(key='Europe/London')),
+    "value_inc_vat": 1,
+    "tariff_code": "E-1R-Test-L",
+    "is_capped": False
+  })
+  expected_rates.append({
+    "start": datetime(2024, 10, 27, 1, 30, fold=1, tzinfo=ZoneInfo(key='Europe/London')),
+    "end": datetime(2024, 10, 27, 2, 0, tzinfo=ZoneInfo(key='Europe/London')),
+    "value_inc_vat": 1,
+    "tariff_code": "E-1R-Test-L",
+    "is_capped": False
+  })
+  expected_rates.extend(create_rate_data(datetime(2024, 10, 27, 2, 0, tzinfo=ZoneInfo(key='Europe/London')), datetime(2024, 10, 29, 0, 0, tzinfo=ZoneInfo(key='Europe/London')), [1]))
+
+  expected_rates_unsorted = expected_rates.copy()
+
+  # Put rates into reverse order to make sure sorting works
+  expected_rates.sort(key=lambda rate: (rate["start"].timestamp(), rate["start"].fold), reverse=True)
+
+  assert expected_rates[-1]["start"] == expected_period_from
+  assert expected_rates[0]["end"] == expected_period_to
+
+  mock_api_called = False
+  requested_period_from = None
+  requested_period_to = None
+  async def async_mocked_get_electricity_rates(*args, **kwargs):
+    nonlocal requested_period_from, requested_period_to, mock_api_called, expected_rates
+
+    requested_client, requested_product_code, requested_tariff_code, is_smart_meter, requested_period_from, requested_period_to = args
+    mock_api_called = True
+    return expected_rates
+
+  actual_fired_events = {}
+  def fire_event(name, metadata):
+    nonlocal actual_fired_events
+    actual_fired_events[name] = metadata
+    return None
+
+  account_info = get_account_info()
+  expected_retrieved_rates = ElectricityRatesCoordinatorResult(current, 1, expected_rates_unsorted)
+  dispatches_result = IntelligentDispatchesCoordinatorResult(dispatches_last_retrieved, 1, IntelligentDispatches([], []))
+
+  with mock.patch.multiple(OctopusEnergyApiClient, async_get_electricity_rates=async_mocked_get_electricity_rates):
+    client = OctopusEnergyApiClient("NOT_REAL")
+    retrieved_rates: ElectricityRatesCoordinatorResult = await async_refresh_electricity_rates_data(
+      current,
+      client,
+      account_info,
+      mpan,
+      serial_number,
+      True,
+      False,
+      existing_rates,
+      None,
+      dispatches_result,
+      True,
+      fire_event
+    )
+
+    assert retrieved_rates is not None
+    assert retrieved_rates.last_retrieved == expected_retrieved_rates.last_retrieved
+    assert retrieved_rates.rates == expected_retrieved_rates.rates
+    assert retrieved_rates.original_rates == expected_retrieved_rates.original_rates
+    assert retrieved_rates.rates_last_adjusted == expected_retrieved_rates.rates_last_adjusted
+    assert mock_api_called == True
+    # assert requested_period_from == expected_period_from
+    # assert requested_period_to == expected_period_to
+
+    assert len(actual_fired_events.keys()) == 3
+    assert_raised_events(actual_fired_events, EVENT_ELECTRICITY_PREVIOUS_DAY_RATES, datetime.strptime("2024-10-26T00:00:00+01:00", "%Y-%m-%dT%H:%M:%S%z"), datetime.strptime("2024-10-27T00:00:00+01:00", "%Y-%m-%dT%H:%M:%S%z"))
+    assert_raised_events(actual_fired_events, EVENT_ELECTRICITY_CURRENT_DAY_RATES, datetime.strptime("2024-10-27T00:00:00+01:00", "%Y-%m-%dT%H:%M:%S%z"), datetime.strptime("2024-10-28T00:00:00+00:00", "%Y-%m-%dT%H:%M:%S%z"))
+    assert_raised_events(actual_fired_events, EVENT_ELECTRICITY_NEXT_DAY_RATES, datetime.strptime("2024-10-28T00:00:00+00:00", "%Y-%m-%dT%H:%M:%S%z"), datetime.strptime("2024-10-29T00:00:00+00:00", "%Y-%m-%dT%H:%M:%S%z"))

tests/unit/target_rates/test_calculate_continuous_times.py

@@ -1163,7 +1163,7 @@ def test_when_multiple_blocks_have_same_value_then_earliest_is_picked():
     }
   ]
 
-  applicable_rates.sort(key=lambda x: x["start"])
+  applicable_rates.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
   result = calculate_continuous_times(
     applicable_rates,
@@ -1193,7 +1193,7 @@ def test_when_weighting_present_with_find_latest_rate_then_latest_time_is_picked
                                            datetime.strptime("2024-10-11T23:30:00+01:00", "%Y-%m-%dT%H:%M:%S%z"),
                                            [0.267159]))
 
-  applicable_rates.sort(key=lambda x: x["start"])
+  applicable_rates.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
   result = calculate_continuous_times(
     applicable_rates,

tests/unit/target_rates/test_calculate_intermittent_times.py

@@ -847,7 +847,7 @@ async def test_when_clocks_go_back_then_correct_rates_are_selected():
     },
   ]
 
-  applicable_rates.sort(key=lambda x: x["start"])
+  applicable_rates.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
   # Act
   result = calculate_intermittent_times(

tests/unit/target_rates/test_get_applicable_rates.py

@@ -602,7 +602,7 @@ async def test_when_clocks_go_back_then_correct_times_are_selected():
     },
   ]
 
-  rates.sort(key=lambda x: x["start"])
+  rates.sort(key=lambda x: (x["start"].timestamp(), x["start"].fold))
 
   # Act
   result = get_applicable_rates(