Log last query_params hit before AirflowTaskTimeout (#1058)

* Log query_params before raising AirflowExceptions * Handle datetimes in queryparams
WordPress · Mar 28, 2023 · 5c2d707 · 5c2d707
1 parent 40461cb
commit 5c2d707
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py b/openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py
@@ -237,8 +237,13 @@ def ingest_records(self, **kwargs) -> None:
                     should_continue = False
 
             except AirflowException as error:
-                # AirflowExceptions should not be caught, as execution should not
-                # continue when the task is being stopped by Airflow.
+                # AirflowExceptions should not be caught or reraised as IngestionErrors,
+                # as execution should not continue when the task is being stopped by
+                # Airflow. However, we should still log the last query_params to be
+                # hit before the error was raised.
+                logger.info(
+                    f"Last query_params used: {json.dumps(query_params, default=str)}"
+                )
 
                 # If errors have already been caught during processing, raise them
                 # as well.