vdk-control-cli: Add support for python_version (#2002)

As part of the initiative to support multiple python versions for data
job deployments, we need to make sure that users can configure their
data jobs to use specific python versions, by setting the
`python_version` property in the job's **config.ini** file.

This change introduces functionality in the vdk-control-cli client to
read the python_version configuration set in the job's config.ini file,
and deploy the job through `vdk deploy ...`

Testing Done: New and existing tests.

Signed-off-by: Andon Andonov <[email protected]>
Co-authored-by: Miroslav Ivanov <[email protected]>

projects/vdk-control-cli/src/vdk/internal/control/command_groups/job/deploy_cli_impl.py

@@ -154,28 +154,29 @@ def update(
         enabled: Optional[bool],  # true, false or None
         job_version: Optional[str],
         vdk_version: Optional[str],
+        python_version: Optional[str] = None,
     ) -> None:
         deployment = DataJobDeployment(enabled=None)
         if job_version:
             deployment.job_version = job_version
         if vdk_version:
             deployment.vdk_version = vdk_version
+        if python_version:
+            deployment.python_version = python_version
         deployment.enabled = enabled
 
-        if job_version:
-            self.__update_job_version(name, team, deployment)
-        elif vdk_version or enabled is not None:
+        if job_version or python_version or vdk_version:
             self.__update_deployment(name, team, deployment)
+        elif enabled is not None:
+            self.__patch_deployment(name, team, deployment)
             msg = f"Deployment of Data Job {name} updated; "
-            if vdk_version:
-                msg = msg + f"vdk version: {vdk_version}; "
             if enabled is not None:
                 msg = msg + "status: " + ("enabled" if enabled else "disabled") + "; "
             log.info(msg)
         else:
             log.warning(f"Nothing to update for deployment of job {name}.")
 
-    def __update_deployment(
+    def __patch_deployment(
         self, name: str, team: str, deployment: DataJobDeployment
     ) -> None:
         log.debug(f"Update Deployment of a job {name} of team {team} : {deployment}")
@@ -186,7 +187,7 @@ def __update_deployment(
             data_job_deployment=deployment,
         )
 
-    def __update_job_version(self, name: str, team: str, deployment: DataJobDeployment):
+    def __update_deployment(self, name: str, team: str, deployment: DataJobDeployment):
         log.debug(
             f"Update Deployment version of a job {name} of team {team} : {deployment}"
         )
@@ -207,6 +208,8 @@ def __update_job_version(self, name: str, team: str, deployment: DataJobDeployme
             result = {
                 "job_name": name,
                 "job_version": deployment.job_version,
+                "vdk_version": deployment.vdk_version,
+                "python_version": deployment.python_version,
             }
             self.__printer.print_dict(result)
 
@@ -233,6 +236,7 @@ def show(self, name: str, team: str) -> None:
                     job_version=d.job_version,
                     last_deployed_by=d.last_deployed_by,
                     last_deployed_date=d.last_deployed_date,
+                    python_version=d.python_version,
                     enabled=d.enabled,
                 ),
                 deployments,
@@ -304,7 +308,16 @@ def create(
                     reason=reason,
                 )
 
+            python_version = job_config.get_python_version()
+
             self.__update_data_job_deploy_configuration(job_path, name, team)
-            self.update(name, team, enabled, data_job_version.version_sha, vdk_version)
+            self.update(
+                name,
+                team,
+                enabled,
+                data_job_version.version_sha,
+                vdk_version,
+                python_version,
+            )
         finally:
             self.__cleanup_archive(archive_path=archive_path)

projects/vdk-control-cli/src/vdk/internal/control/job/job_config.py

@@ -52,6 +52,9 @@ def set_team_if_exists(self, value) -> None:
     def get_schedule_cron(self) -> str:
         return self._get_value("job", "schedule_cron")
 
+    def get_python_version(self) -> str:
+        return str(self._get_value("job", "python_version"))
+
     def get_enable_execution_notifications(self) -> bool:
         return self._get_boolean(
             "contacts", "enable_execution_notifications", fallback=True

projects/vdk-control-cli/tests/resources/test-job-python-version/config.ini

@@ -0,0 +1,10 @@
+[owner]
+team = test-team
+[job]
+schedule_cron = 16 11 * * *
+python_version = 3.8
+[contacts]
+
+notified_on_job_failure_user_error[email protected]
+
+notified_on_job_success[email protected]

projects/vdk-control-cli/tests/vdk/internal/control/command_groups/job/test_deploy.py

@@ -156,9 +156,30 @@ def test_deploy_enable_disable(httpserver: PluginHTTPServer, tmpdir: LocalPath):
     assert httpserver.log[1][0].data == b'{"enabled": false}'
 
 
+def test_deploy_python_version_in_config(
+    httpserver: PluginHTTPServer, tmpdir: LocalPath
+):
+    _, deploy_args = prepare_new_deploy(httpserver, "test-job-python-version")
+
+    runner = CliRunner()
+    result = runner.invoke(deploy, deploy_args)
+    test_utils.assert_click_status(result, 0)
+
+    test_job_path = find_test_resource("test-job-python-version")
+    assert os.path.exists(f"{test_job_path}.zip") is False
+
+    posted_data = json.loads(httpserver.log[3][0].data)
+    assert posted_data["job_version"] is not None
+    assert posted_data["python_version"] == "3.8"
+
+
 def test_set_vdk_version(httpserver: PluginHTTPServer, tmpdir: LocalPath):
     rest_api_url = httpserver.url_for("")
 
+    httpserver.expect_request(
+        uri="/data-jobs/for-team/test-team/jobs/test-job/deployments",
+        method="POST",
+    ).respond_with_response(Response(status=202))
     httpserver.expect_request(
         uri=f"/data-jobs/for-team/test-team/jobs/test-job/deployments/{DEPLOYMENT_ID}",
         method="PATCH",
@@ -180,6 +201,8 @@ def test_set_vdk_version(httpserver: PluginHTTPServer, tmpdir: LocalPath):
         ],
     )
     test_utils.assert_click_status(result, 0)
+    assert len(httpserver.log) == 1
+    assert httpserver.log[0][0].method == "POST"
     assert httpserver.log[0][0].data == b'{"vdk_version": "1.1.1"}'
 
 
@@ -221,9 +244,9 @@ def test_update_miltiple(httpserver: PluginHTTPServer, tmpdir: LocalPath):
     rest_api_url = httpserver.url_for("")
 
     httpserver.expect_request(
-        uri=f"/data-jobs/for-team/test-team/jobs/test-job/deployments/{DEPLOYMENT_ID}",
-        method="PATCH",
-    ).respond_with_response(Response(status=200))
+        uri=f"/data-jobs/for-team/test-team/jobs/test-job/deployments",
+        method="POST",
+    ).respond_with_response(Response(status=202))
 
     runner = CliRunner()
     result = runner.invoke(
@@ -305,6 +328,7 @@ def test_deploy_show_with_json_output(httpserver: PluginHTTPServer, tmpdir: Loca
             "mode": "testing",
             "last_deployed_by": "user",
             "last_deployed_date": "2021-02-25T09:16:53.323Z",
+            "python_version": "3.8",
         }
     ]
     httpserver.expect_request(
@@ -343,6 +367,9 @@ def test_deploy_show_with_json_output(httpserver: PluginHTTPServer, tmpdir: Loca
     assert (
         list(json_result)[0]["last_deployed_date"] == "2021-02-25T09:16:53.323Z"
     ), f"expected data not found in output: {result.output}"
+    assert (
+        list(json_result)[0]["python_version"] == "3.8"
+    ), f"expected data not found in output: {result.output}"
 
 
 def test_deploy_show_with_json_output_and_no_deployments(httpserver: PluginHTTPServer):
@@ -452,25 +479,25 @@ def test_deploy_failed_team_validation(httpserver: PluginHTTPServer, tmpdir: Loc
     assert "what" in result.output and "why" in result.output
 
 
-def prepare_new_deploy(httpserver) -> Tuple[str, str]:
+def prepare_new_deploy(httpserver, job_name: str = "test-job") -> Tuple[str, str]:
     rest_api_url = httpserver.url_for("")
-    mock_base_requests(httpserver)
+    mock_base_requests(httpserver, job_name)
     job_version = DataJobVersion(version_sha="17012900f60461778c01ab24728807e70a5f2c87")
     httpserver.expect_request(
-        uri="/data-jobs/for-team/test-team/jobs/test-job/sources",
+        uri=f"/data-jobs/for-team/test-team/jobs/{job_name}/sources",
         method="POST",
         headers={"Content-Type": "application/octet-stream"},
         query_string="reason=reason",
     ).respond_with_json(job_version.to_dict())
-    test_job_path = find_test_resource("test-job")
+    test_job_path = find_test_resource(job_name)
 
     open(f"{test_job_path}.zip", "w")
 
     assert os.path.exists(f"{test_job_path}.zip") is True
 
     deploy_args = [
         "-n",
-        "test-job",
+        job_name,
         "-t",
         "test-team",
         "-p",
@@ -484,21 +511,21 @@ def prepare_new_deploy(httpserver) -> Tuple[str, str]:
     return job_version, deploy_args
 
 
-def mock_base_requests(httpserver):
+def mock_base_requests(httpserver, job_name: str = "test-job"):
     existing_job = DataJob(
-        job_name="test-job", team="test-team", description="", config=DataJobConfig()
+        job_name=job_name, team="test-team", description="", config=DataJobConfig()
     )
     httpserver.expect_request(
-        uri="/data-jobs/for-team/test-team/jobs/test-job", method="GET"
+        uri=f"/data-jobs/for-team/test-team/jobs/{job_name}", method="GET"
     ).respond_with_json(existing_job.to_dict())
 
     httpserver.expect_request(
-        uri="/data-jobs/for-team/test-team/jobs/test-job", method="PUT"
+        uri=f"/data-jobs/for-team/test-team/jobs/{job_name}", method="PUT"
     ).respond_with_response(Response(status=200))
 
     httpserver.expect_request(
-        uri="/data-jobs/for-team/test-team/jobs/test-job/deployments", method="POST"
+        uri=f"/data-jobs/for-team/test-team/jobs/{job_name}/deployments", method="POST"
     ).respond_with_response(Response(status=200))
     httpserver.expect_request(
-        uri=f"/data-jobs/for-team/test-team/jobs/test-job/deployments/{DEPLOYMENT_ID}"
+        uri=f"/data-jobs/for-team/test-team/jobs/{job_name}/deployments/{DEPLOYMENT_ID}"
     ).respond_with_response(Response(status=200))