From c3f37323372847395d8a2b0d031c78fd7c95a836 Mon Sep 17 00:00:00 2001 From: xs2tarunkukreja <39728640+xs2tarunkukreja@users.noreply.github.com> Date: Tue, 17 Oct 2023 19:27:54 +0530 Subject: [PATCH] It is failing when we are passing the dictionary. As per documentation, it should also accept YAML and JSON directly. The operator is working fine when we pass the file path. But fails when we pass the YAML or JSON as dictionary. --- .../providers/cncf/kubernetes/operators/spark_kubernetes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index 82288ef631bca..87efd5a680e94 100644 --- a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -111,6 +111,11 @@ def _get_namespace_event_stream(self, namespace, query_kwargs=None): raise def execute(self, context: Context): + if isinstance(self.application_file, str): + body = _load_body_to_dict(self.application_file) + else: + body = self.application_file + body = _load_body_to_dict(self.application_file) name = body["metadata"]["name"] namespace = self.namespace or self.hook.get_namespace()