Fix inconsistent spark driver and executor image (#3996)

* Fix inconsistent spark driver and executor image

* Add docker registry url

Author: chi-yelp

paasta_tools/tron_tools.py

@@ -348,8 +348,6 @@ def build_spark_config(self) -> Dict[str, str]:
             f"tron_spark_{self.get_service()}_{self.get_instance()}",
         )
 
-        docker_img_url = self.get_docker_url(system_paasta_config)
-
         spark_conf_builder = SparkConfBuilder(is_driver_on_k8s_tron=True)
         spark_conf = spark_conf_builder.get_spark_conf(
             cluster_manager="kubernetes",
@@ -359,7 +357,7 @@ def build_spark_config(self) -> Dict[str, str]:
             paasta_pool=self.get_spark_executor_pool(),
             paasta_service=self.get_service(),
             paasta_instance=self.get_instance(),
-            docker_img=docker_img_url,
+            docker_img=f"{self.get_docker_registry()}/$PAASTA_DOCKER_IMAGE",
             extra_volumes=self.get_volumes(
                 system_paasta_config.get_volumes(),
                 uses_bulkdata_default=system_paasta_config.get_uses_bulkdata_default(),

tests/test_tron_tools.py

@@ -1322,7 +1322,7 @@ def test_format_tron_action_dict_spark(
             "--conf spark.executorEnv.PAASTA_INSTANCE_TYPE=spark "
             "--conf spark.executorEnv.SPARK_EXECUTOR_DIRS=/tmp "
             "--conf spark.kubernetes.pyspark.pythonVersion=3 "
-            "--conf spark.kubernetes.container.image=docker-registry.com:400/my_service:paasta-123abcde "
+            "--conf spark.kubernetes.container.image=docker-registry.com:400/$PAASTA_DOCKER_IMAGE "
             "--conf spark.kubernetes.namespace=paasta-spark "
             "--conf spark.kubernetes.executor.label.yelp.com/paasta_service=my_service "
             "--conf spark.kubernetes.executor.label.yelp.com/paasta_instance=my_job.do_something "