From c273fd1f21a6870fb81b7e557dd46f2d585a9535 Mon Sep 17 00:00:00 2001 From: Andy <87700128+doks5@users.noreply.github.com> Date: Tue, 2 May 2023 12:46:10 +0300 Subject: [PATCH] vdk-control-cli: Add support for python_version (#2002) As part of the initiative to support multiple python versions for data job deployments, we need to make sure that users can configure their data jobs to use specific python versions, by setting the `python_version` property in the job's **config.ini** file. This change introduces functionality in the vdk-control-cli client to read the python_version configuration set in the job's config.ini file, and deploy the job through `vdk deploy ...` Testing Done: New and existing tests. Signed-off-by: Andon Andonov Co-authored-by: Miroslav Ivanov <40535952+mivanov1988@users.noreply.github.com> --- .../command_groups/job/deploy_cli_impl.py | 29 ++++++--- .../vdk/internal/control/job/job_config.py | 3 + .../job-zip/test-job-python-version.zip | Bin 0 -> 567 bytes .../test-job-python-version/config.ini | 10 ++++ .../control/command_groups/job/test_deploy.py | 55 +++++++++++++----- 5 files changed, 75 insertions(+), 22 deletions(-) create mode 100644 projects/vdk-control-cli/tests/resources/job-zip/test-job-python-version.zip create mode 100644 projects/vdk-control-cli/tests/resources/test-job-python-version/config.ini diff --git a/projects/vdk-control-cli/src/vdk/internal/control/command_groups/job/deploy_cli_impl.py b/projects/vdk-control-cli/src/vdk/internal/control/command_groups/job/deploy_cli_impl.py index 3ca8137ac9..f088f2b749 100644 --- a/projects/vdk-control-cli/src/vdk/internal/control/command_groups/job/deploy_cli_impl.py +++ b/projects/vdk-control-cli/src/vdk/internal/control/command_groups/job/deploy_cli_impl.py @@ -154,28 +154,29 @@ def update( enabled: Optional[bool], # true, false or None job_version: Optional[str], vdk_version: Optional[str], + python_version: Optional[str] = None, ) -> None: deployment = DataJobDeployment(enabled=None) if job_version: deployment.job_version = job_version if vdk_version: deployment.vdk_version = vdk_version + if python_version: + deployment.python_version = python_version deployment.enabled = enabled - if job_version: - self.__update_job_version(name, team, deployment) - elif vdk_version or enabled is not None: + if job_version or python_version or vdk_version: self.__update_deployment(name, team, deployment) + elif enabled is not None: + self.__patch_deployment(name, team, deployment) msg = f"Deployment of Data Job {name} updated; " - if vdk_version: - msg = msg + f"vdk version: {vdk_version}; " if enabled is not None: msg = msg + "status: " + ("enabled" if enabled else "disabled") + "; " log.info(msg) else: log.warning(f"Nothing to update for deployment of job {name}.") - def __update_deployment( + def __patch_deployment( self, name: str, team: str, deployment: DataJobDeployment ) -> None: log.debug(f"Update Deployment of a job {name} of team {team} : {deployment}") @@ -186,7 +187,7 @@ def __update_deployment( data_job_deployment=deployment, ) - def __update_job_version(self, name: str, team: str, deployment: DataJobDeployment): + def __update_deployment(self, name: str, team: str, deployment: DataJobDeployment): log.debug( f"Update Deployment version of a job {name} of team {team} : {deployment}" ) @@ -207,6 +208,8 @@ def __update_job_version(self, name: str, team: str, deployment: DataJobDeployme result = { "job_name": name, "job_version": deployment.job_version, + "vdk_version": deployment.vdk_version, + "python_version": deployment.python_version, } self.__printer.print_dict(result) @@ -233,6 +236,7 @@ def show(self, name: str, team: str) -> None: job_version=d.job_version, last_deployed_by=d.last_deployed_by, last_deployed_date=d.last_deployed_date, + python_version=d.python_version, enabled=d.enabled, ), deployments, @@ -304,7 +308,16 @@ def create( reason=reason, ) + python_version = job_config.get_python_version() + self.__update_data_job_deploy_configuration(job_path, name, team) - self.update(name, team, enabled, data_job_version.version_sha, vdk_version) + self.update( + name, + team, + enabled, + data_job_version.version_sha, + vdk_version, + python_version, + ) finally: self.__cleanup_archive(archive_path=archive_path) diff --git a/projects/vdk-control-cli/src/vdk/internal/control/job/job_config.py b/projects/vdk-control-cli/src/vdk/internal/control/job/job_config.py index 6be4967535..7acd567df4 100644 --- a/projects/vdk-control-cli/src/vdk/internal/control/job/job_config.py +++ b/projects/vdk-control-cli/src/vdk/internal/control/job/job_config.py @@ -52,6 +52,9 @@ def set_team_if_exists(self, value) -> None: def get_schedule_cron(self) -> str: return self._get_value("job", "schedule_cron") + def get_python_version(self) -> str: + return str(self._get_value("job", "python_version")) + def get_enable_execution_notifications(self) -> bool: return self._get_boolean( "contacts", "enable_execution_notifications", fallback=True diff --git a/projects/vdk-control-cli/tests/resources/job-zip/test-job-python-version.zip b/projects/vdk-control-cli/tests/resources/job-zip/test-job-python-version.zip new file mode 100644 index 0000000000000000000000000000000000000000..2c03e3c8b84ae9b55f63d680bc3b8197dea15f78 GIT binary patch literal 567 zcmWIWW@Zs#00GP3g<)U@l#pOhU?@o~F44`(Ptq-@EXl~v(=AIaD$dN$(+>^dWniz* z^iOF7Vh~+g!Og(P@`9Ox0ZasdP32(VKr!_I&}1b%rY7g-rDdk;W#(nV&38z{YJP6- zL9Rmv0xs`=i&}Dvbt%Mg{d3n)%-e2qDJ8Z1@}v8=D?* Tuple[str, str]: +def prepare_new_deploy(httpserver, job_name: str = "test-job") -> Tuple[str, str]: rest_api_url = httpserver.url_for("") - mock_base_requests(httpserver) + mock_base_requests(httpserver, job_name) job_version = DataJobVersion(version_sha="17012900f60461778c01ab24728807e70a5f2c87") httpserver.expect_request( - uri="/data-jobs/for-team/test-team/jobs/test-job/sources", + uri=f"/data-jobs/for-team/test-team/jobs/{job_name}/sources", method="POST", headers={"Content-Type": "application/octet-stream"}, query_string="reason=reason", ).respond_with_json(job_version.to_dict()) - test_job_path = find_test_resource("test-job") + test_job_path = find_test_resource(job_name) open(f"{test_job_path}.zip", "w") @@ -470,7 +497,7 @@ def prepare_new_deploy(httpserver) -> Tuple[str, str]: deploy_args = [ "-n", - "test-job", + job_name, "-t", "test-team", "-p", @@ -484,21 +511,21 @@ def prepare_new_deploy(httpserver) -> Tuple[str, str]: return job_version, deploy_args -def mock_base_requests(httpserver): +def mock_base_requests(httpserver, job_name: str = "test-job"): existing_job = DataJob( - job_name="test-job", team="test-team", description="", config=DataJobConfig() + job_name=job_name, team="test-team", description="", config=DataJobConfig() ) httpserver.expect_request( - uri="/data-jobs/for-team/test-team/jobs/test-job", method="GET" + uri=f"/data-jobs/for-team/test-team/jobs/{job_name}", method="GET" ).respond_with_json(existing_job.to_dict()) httpserver.expect_request( - uri="/data-jobs/for-team/test-team/jobs/test-job", method="PUT" + uri=f"/data-jobs/for-team/test-team/jobs/{job_name}", method="PUT" ).respond_with_response(Response(status=200)) httpserver.expect_request( - uri="/data-jobs/for-team/test-team/jobs/test-job/deployments", method="POST" + uri=f"/data-jobs/for-team/test-team/jobs/{job_name}/deployments", method="POST" ).respond_with_response(Response(status=200)) httpserver.expect_request( - uri=f"/data-jobs/for-team/test-team/jobs/test-job/deployments/{DEPLOYMENT_ID}" + uri=f"/data-jobs/for-team/test-team/jobs/{job_name}/deployments/{DEPLOYMENT_ID}" ).respond_with_response(Response(status=200))