From 59866b0fefc8aeea75326d0d68914e9f8184116a Mon Sep 17 00:00:00 2001 From: dirrao Date: Wed, 22 May 2024 19:25:50 +0530 Subject: [PATCH] Resolving emr deprecated warnings --- airflow/providers/amazon/aws/operators/emr.py | 25 +++++++++++-------- tests/always/test_example_dags.py | 1 - 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/emr.py b/airflow/providers/amazon/aws/operators/emr.py index ec1be30f91346..01bf61b82a699 100644 --- a/airflow/providers/amazon/aws/operators/emr.py +++ b/airflow/providers/amazon/aws/operators/emr.py @@ -734,10 +734,21 @@ def __init__( waiter_max_attempts: int | None = None, waiter_delay: int | None = None, waiter_countdown: int | None = None, - waiter_check_interval_seconds: int = 60, + waiter_check_interval_seconds: int, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ): + if waiter_check_interval_seconds: + warnings.warn( + "The parameter waiter_check_interval_seconds has been deprecated to " + "standardize naming conventions. Please use waiter_delay instead. In the " + "future this will default to None and defer to the waiter's default value.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = waiter_check_interval_seconds + else: + waiter_check_interval_seconds = 60 if waiter_countdown: warnings.warn( "The parameter waiter_countdown has been deprecated to standardize " @@ -749,15 +760,7 @@ def __init__( # waiter_countdown defaults to never timing out, which is not supported # by boto waiters, so we will set it here to "a very long time" for now. waiter_max_attempts = (waiter_countdown or 999) // waiter_check_interval_seconds - if waiter_check_interval_seconds: - warnings.warn( - "The parameter waiter_check_interval_seconds has been deprecated to " - "standardize naming conventions. Please use waiter_delay instead. In the " - "future this will default to None and defer to the waiter's default value.", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - waiter_delay = waiter_check_interval_seconds + super().__init__(**kwargs) self.aws_conn_id = aws_conn_id self.emr_conn_id = emr_conn_id @@ -765,7 +768,7 @@ def __init__( self.region_name = region_name self.wait_for_completion = wait_for_completion self.waiter_max_attempts = waiter_max_attempts or 60 - self.waiter_delay = waiter_delay or 30 + self.waiter_delay = waiter_delay or 60 self.deferrable = deferrable @cached_property diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 95083dad1f430..eaa4be6edcc12 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -49,7 +49,6 @@ # and a corresponding Issue should be created on GitHub. "tests/system/providers/amazon/aws/example_ecs_fargate.py", "tests/system/providers/amazon/aws/example_eks_with_nodegroups.py", - "tests/system/providers/amazon/aws/example_emr.py", "tests/system/providers/amazon/aws/example_emr_notebook_execution.py", "tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py", "tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py",