diff --git a/projects/control-service/projects/model/apidefs/datajob-api/api.yaml b/projects/control-service/projects/model/apidefs/datajob-api/api.yaml index d069f51e02..750736a1ec 100644 --- a/projects/control-service/projects/model/apidefs/datajob-api/api.yaml +++ b/projects/control-service/projects/model/apidefs/datajob-api/api.yaml @@ -79,7 +79,7 @@ paths: List data Jobs with GraphQL like query. By choosing which field to be returned you can control the output. You can learn more about the GraphQL queries by visiting [GraphQL official website](https://graphql.org/learn/queries/) Query should be provided as GET parameter, not by POST body. Don't worry about the spaces and tabs - Keep in mind that each additional field could make query response time slower, for instance deployments, + Keep in mind that each aditional field could make query response time slower, for instance deployments, it's best if you request only what you need

The pageNumber and pageSize arguments are required! Page number should be a number greater than 1, @@ -102,7 +102,7 @@ paths: an invalid response will be return.
* pattern should be a non-empty string which the provided property should contains [ignoring cases], for instance: `starshot` pattern will match import-starshot-sql, StarShot-servers and notify-starshot job names, but it won't match stars-shot-daily-prune` - If a pattern string is not provided, then you must at least provide the property field
+ If a pattern string is not provided, then you must atleast provide the property field
* sort should be an enum value - ASC (ascending) or DESC (descending) option [not required, default is ASC] Multiple filters could be applied, but maximum one should contain sorting! ``` @@ -376,8 +376,8 @@ paths:
  • 5 characters at least
  • should contain only lowercase alphanumeric symbols and dash [a-z0-9\-]

    - When successfully created clients can download the keytab associated with the Data Job to run it locally.
    - To deploy so that it can be executed regularly in the runtime environment use /data-jobs/for-team/{team_name}/jobs/{job_name}/deployments API. + When succesfully created clients can download the keytab associated with the Data Job to run it locally.
    + To deploy so that it can be executed reguarly in the runtime environment use /data-jobs/for-team/{team_name}/jobs/{job_name}/deployments API. '/data-jobs/for-team/{team_name}/jobs/{job_name}': summary: | @@ -548,9 +548,9 @@ paths: parameters: - name: send_notification description: | - Allows administrators to disable sending notification during maintenance window for platform errors - + Allows administrators to disable sending notification during maintance window for platfrom errors - e.g if all jobs are being re-deployed due to migration or after recovering from an outage. - Monitoring data will still be populated so deployment status can be monitored by admins. + Monitoring data will still be popoulated so deployment status can be monitored by admins. schema: type: boolean default: True @@ -569,7 +569,7 @@ paths: summary: Creates or updates a deployment of a Data Job. | (Stable) description: | (Introduced in v1.0) | Creates or updates a deployment of a Data Job. - The API returns before the actual Job is deployed (the operation is asynchronous). + The API returns before the actual Job is deployed (the operation is asynchrounous). Depending on settings specified during Data Job creation, clients will get an e-mail in case of success or failure of the deployment. If you need to change only deployment settings without changing the job version, use PATCH deployments request which is synchronous. parameters: @@ -632,7 +632,7 @@ paths: summary: | Patch a deployment of a Data Job. Use it to change the configuration of a data job. For example: to enable or disable deployment, to change the vdk version. - The operation is guaranteed to be synchronous so it cannot be used to deploy new version of a data job - + The operation is guranteed to be synchrounous so it cannot be used to deploy new version of a data job - job_version cannot be changed using PATCH. Use POST .../deployments for this. | (Stable) description: (Introduced in v1.0) parameters: @@ -818,7 +818,7 @@ paths: operationId: dataJobLogsDownload summary: | Download data job logs. - This API is guaranteed to provide logs only if the jobs is currently running. + This API is guranteed to provide logs only if the jobs is currently running. For logs from older job executions - use logsUrl field passed by GET execution API or jobsQuery API. description: (Introduced in v1.2.9) parameters: @@ -902,7 +902,7 @@ paths: '/data-jobs/for-team/{team_name}/jobs/{job_name}/deployments/{deployment_id}/secrets': summary: | - Data Job Secrets API allows the management of secrets for specific data jobs. Secrets are any sensitive data: + Data Job Secrets API allows the management of secrets for specific data jobs. Secreats are any sensitive data: passwords/tokens/credentials which are need by a data job. get: tags: @@ -1134,10 +1134,10 @@ components: description: A deployment of the Data Job type: object properties: - vdk_image: - description: A specific VDK image to use + vdk_version: + description: A specific VDK version to use type: string - example: "example.com/versatiledatakit/vdk:latest" + example: 2.1 job_version: description: Job version (can be Git commit) type: string @@ -1171,10 +1171,10 @@ components: description: A deployment status of the Data Job, which includes information such as last deployment date, and who deployed the Data Job. type: object properties: - vdk_image: - description: A specific VDK image to use + vdk_version: + description: A specific VDK version to use type: string - example: "example.com/versatiledatakit/vdk:latest" + example: 2.1 job_version: description: Job version (can be Git commit) type: string @@ -1333,7 +1333,7 @@ components: example: release DataJobResources: - description: Resource configuration of a data Data Job Deployment. + description: Resource cofiguration of a data Data Job Deployment. type: object properties: cpu_request: diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobDeploymentCrudIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobDeploymentCrudIT.java index 8e3a689809..5e0c34debe 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobDeploymentCrudIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobDeploymentCrudIT.java @@ -41,7 +41,7 @@ private void setVdkVersionForDeployment() throws Exception { .with(user("user")) .content(getDataJobDeploymentVdkVersionRequestBody("new_vdk_version_tag")) .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()); + .andExpect(status().isBadRequest()); } private void disableDeployment() throws Exception { @@ -67,7 +67,7 @@ private void verifyVersion() throws Exception { .with(user("user")) .contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.vdk_image", is("new_vdk_version_tag"))); + .andExpect(jsonPath("$.vdk_version", is("release"))); } private void resetVdkDeploymentVersion() throws Exception { @@ -80,7 +80,7 @@ private void resetVdkDeploymentVersion() throws Exception { .with(user("user")) .content(getDataJobDeploymentVdkVersionRequestBody("")) .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()); + .andExpect(status().isBadRequest()); } private MvcResult getDeployment() throws Exception { @@ -105,9 +105,9 @@ private void checkDeployment() throws Exception { Assertions.assertEquals("user", jobDeployment.getLastDeployedBy()); Assertions.assertEquals("3.9", jobDeployment.getPythonVersion()); Assertions.assertFalse(jobDeployment.getEnabled()); - Assertions.assertEquals("new_vdk_version_tag", jobDeployment.getVdkImage()); + Assertions.assertEquals("release", jobDeployment.getVdkVersion()); Assertions.assertNotNull(jobDeployment.getJobVersion()); - Assertions.assertNotNull(jobDeployment.getVdkImage()); + Assertions.assertNotNull(jobDeployment.getVdkVersion()); } @Override diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/TestJobImageBuilderDynamicVdkImageIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/TestJobImageBuilderDynamicVdkImageIT.java index ceb94ee325..de5e775fad 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/TestJobImageBuilderDynamicVdkImageIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/TestJobImageBuilderDynamicVdkImageIT.java @@ -178,9 +178,7 @@ public void testDataJobDeploymentDynamicVdkVersion() throws Exception { Assertions.assertEquals(true, jobDeployment.getEnabled()); // by default the version is the same as the tag specified by datajobs.vdk.image // for integration test this is registry.hub.docker.com/versatiledatakit/quickstart-vdk:release - Assertions.assertEquals( - "ghcr.io/versatile-data-kit-dev/versatiledatakit/quickstart-vdk:release", - jobDeployment.getVdkImage()); + Assertions.assertEquals("release", jobDeployment.getVdkVersion()); Assertions.assertEquals("user", jobDeployment.getLastDeployedBy()); // just check some valid date is returned. It would be too error-prone/brittle to verify exact // time. @@ -214,9 +212,9 @@ public void testDataJobDeploymentDynamicVdkVersion() throws Exception { .with(user("user")) .content(getDataJobDeploymentVdkVersionRequestBody("new_vdk_version_tag")) .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()); + .andExpect(status().isBadRequest()); - // verify vdk version is changed + // verify vdk version is not changed mockMvc .perform( get(String.format( @@ -225,7 +223,7 @@ public void testDataJobDeploymentDynamicVdkVersion() throws Exception { .with(user("user")) .contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.vdk_image", is("new_vdk_version_tag"))); + .andExpect(jsonPath("$.vdk_version", is("release"))); // Execute change python version and set corresponding vdk version for deployment mockMvc @@ -247,7 +245,7 @@ public void testDataJobDeploymentDynamicVdkVersion() throws Exception { Assertions.assertEquals(false, cronJob.getEnabled()); Assertions.assertTrue(cronJob.getImageName().endsWith(testJobVersionSha)); Assertions.assertEquals("user", cronJob.getLastDeployedBy()); - Assertions.assertTrue(cronJob.getVdkImage().endsWith("pre-release")); + Assertions.assertTrue(cronJob.getVdkVersion().endsWith("pre-release")); Assertions.assertEquals("3.8", cronJob.getPythonVersion()); // Execute delete deployment diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 14cdf48697..1ae111c32c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -189,7 +189,7 @@ public String getDataJobDeploymentEnableRequestBody(boolean enabled) public String getDataJobDeploymentVdkVersionRequestBody(String vdkVersion) throws JsonProcessingException { var deployment = new DataJobDeployment(); - deployment.setVdkImage(vdkVersion); + deployment.setVdkVersion(vdkVersion); return mapper.writeValueAsString(deployment); } } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DeploymentModelConverter.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DeploymentModelConverter.java index 18123dc286..b9296a4c18 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DeploymentModelConverter.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DeploymentModelConverter.java @@ -36,7 +36,7 @@ public static JobDeployment toJobDeployment( deployment.setResources(jobDeploymentStatus.getResources()); deployment.setMode(jobDeploymentStatus.getMode()); deployment.setGitCommitSha(jobDeploymentStatus.getGitCommitSha()); - deployment.setVdkImage(jobDeploymentStatus.getVdkImage()); + deployment.setVdkVersion(jobDeploymentStatus.getVdkVersion()); deployment.setPythonVersion(jobDeploymentStatus.getPythonVersion()); return deployment; @@ -62,7 +62,7 @@ public static DesiredDataJobDeployment toDesiredDataJobDeployment(JobDeployment deployment.setGitCommitSha(jobDeployment.getGitCommitSha()); deployment.setPythonVersion(jobDeployment.getPythonVersion()); - deployment.setVdkImage(jobDeployment.getVdkImage()); + deployment.setVdkVersion(jobDeployment.getVdkVersion()); return deployment; } @@ -165,10 +165,10 @@ public static JobDeployment mergeDeployments( newDeployment.getGitCommitSha() != null ? newDeployment.getGitCommitSha() : oldDeployment.getGitCommitSha()); - mergedDeployment.setVdkImage( - (newDeployment.getVdkImage() != null || newDeployment.getPythonVersion() != null) - ? newDeployment.getVdkImage() - : oldDeployment.getVdkImage()); + mergedDeployment.setVdkVersion( + newDeployment.getVdkVersion() != null + ? newDeployment.getVdkVersion() + : oldDeployment.getVdkVersion()); mergedDeployment.setPythonVersion( newDeployment.getPythonVersion() != null ? newDeployment.getPythonVersion() @@ -223,6 +223,10 @@ public static DesiredDataJobDeployment mergeDeployments( newDeployment.getPythonVersion() != null ? newDeployment.getPythonVersion() : oldDeployment.getPythonVersion()); + mergedDeployment.setVdkVersion( + newDeployment.getVdkVersion() != null + ? newDeployment.getVdkVersion() + : oldDeployment.getVdkVersion()); mergedDeployment.setLastDeployedBy( userDeployer != null ? userDeployer : oldDeployment.getLastDeployedBy()); mergedDeployment.setSchedule( @@ -236,10 +240,6 @@ public static DesiredDataJobDeployment mergeDeployments( newDeployment.getEnabled() != null ? newDeployment.getEnabled() : oldDeployment.getEnabled()); - mergedDeployment.setVdkImage( - (newDeployment.getVdkImage() != null || newDeployment.getPythonVersion() != null) - ? newDeployment.getVdkImage() - : oldDeployment.getVdkImage()); return mergedDeployment; } @@ -316,6 +316,7 @@ public static DataJobDeploymentStatus toJobDeploymentStatus( var deploymentStatus = new DataJobDeploymentStatus(); deploymentStatus.setJobVersion(actualDataJobDeployment.getGitCommitSha()); deploymentStatus.setPythonVersion(actualDataJobDeployment.getPythonVersion()); + deploymentStatus.setVdkVersion(actualDataJobDeployment.getVdkVersion()); deploymentStatus.setId(actualDataJobDeployment.getDataJobName()); deploymentStatus.setEnabled(actualDataJobDeployment.getEnabled()); deploymentStatus.setContacts(getContactsFromJob(job)); @@ -328,7 +329,6 @@ public static DataJobDeploymentStatus toJobDeploymentStatus( ? null : actualDataJobDeployment.getLastDeployedDate().toString()); deploymentStatus.setLastDeployedBy(actualDataJobDeployment.getLastDeployedBy()); - deploymentStatus.setVdkImage(actualDataJobDeployment.getVdkImage()); return deploymentStatus; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToApiModelConverter.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToApiModelConverter.java index be65b57f2d..a5aa87c382 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToApiModelConverter.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToApiModelConverter.java @@ -109,7 +109,7 @@ public static DataJobDeploymentStatus toDataJobDeploymentStatus( deployment.setJobVersion(jobDeploymentStatus.getGitCommitSha()); deployment.setLastDeployedBy(jobDeploymentStatus.getLastDeployedBy()); deployment.setLastDeployedDate(jobDeploymentStatus.getLastDeployedDate()); - deployment.setVdkImage(jobDeploymentStatus.getVdkImage()); + deployment.setVdkVersion(jobDeploymentStatus.getVdkVersion()); deployment.setPythonVersion(jobDeploymentStatus.getPythonVersion()); return deployment; @@ -181,7 +181,7 @@ public static V2DataJobDeployment toV2DataJobDeployment( // TODO finish mapping implementation in TAUR-1535 v2DataJobDeployment.setContacts(new DataJobContacts()); v2DataJobDeployment.setSchedule(new V2DataJobSchedule()); - v2DataJobDeployment.setVdkVersion(jobDeploymentStatus.getVdkImage()); + v2DataJobDeployment.setVdkVersion(jobDeploymentStatus.getVdkVersion()); v2DataJobDeployment.setExecutions(new ArrayList<>()); return v2DataJobDeployment; @@ -202,7 +202,7 @@ public static DataJobExecution jobExecutionToConvert( .logsUrl(logsUrl) .deployment( new DataJobDeployment() - .vdkImage(jobExecutionToConvert.getVdkImage()) + .vdkVersion(jobExecutionToConvert.getVdkVersion()) .jobVersion(jobExecutionToConvert.getJobVersion()) .pythonVersion(jobExecutionToConvert.getJobPythonVersion()) .schedule( diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToModelApiConverter.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToModelApiConverter.java index da813b242c..bbb272f561 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToModelApiConverter.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToModelApiConverter.java @@ -25,7 +25,7 @@ public static JobDeployment toJobDeployment( jobDeployment.setMode(dataJobDeployment.getMode().toString()); } jobDeployment.setGitCommitSha(dataJobDeployment.getJobVersion()); - jobDeployment.setVdkImage(dataJobDeployment.getVdkImage()); + jobDeployment.setVdkVersion(dataJobDeployment.getVdkVersion()); if (dataJobDeployment.getPythonVersion() != null) { jobDeployment.setPythonVersion(dataJobDeployment.getPythonVersion()); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 9e21a6a6e2..028a917874 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -2004,13 +2004,9 @@ private Optional mapV1beta1CronJobToDeploymentStatus( var initContainers = cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getSpec().getInitContainers(); if (!CollectionUtils.isEmpty(initContainers)) { - // With the functionality to pass custom vdk images that are different from the ones - // specified - // in the supportedPythonVersions, we need to pass the whole image as a VdkVersion instead - // of - // just the image tag. String vdkImage = initContainers.get(0).getImage(); - deployment.setVdkImage(vdkImage); + deployment.setVdkImageName(vdkImage); + deployment.setVdkVersion(DockerImageName.getTag(vdkImage)); } else { log.warn("Missing init container for cronjob {}", cronJobName); } @@ -2070,13 +2066,9 @@ private Optional mapV1CronJobToDeploymentStatus( var initContainers = cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getSpec().getInitContainers(); if (!CollectionUtils.isEmpty(initContainers)) { - // With the functionality to pass custom vdk images that are different from the ones - // specified - // in the supportedPythonVersions, we need to pass the whole image as a VdkVersion instead - // of - // just the image tag. String vdkImage = initContainers.get(0).getImage(); - deployment.setVdkImage(vdkImage); + deployment.setVdkImageName(vdkImage); + deployment.setVdkVersion(DockerImageName.getTag(vdkImage)); } else { log.warn("Missing init container for cronjob {}", cronJobName); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java index b1f4d8491f..98d8c79f95 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java @@ -254,9 +254,8 @@ private void updateCronJob(DataJob dataJob, JobDeployment jobDeployment, String "-c", "cp -r $(python -c \"from distutils.sysconfig import get_python_lib;" + " print(get_python_lib())\") /vdk/. && cp /usr/local/bin/vdk /vdk/."); - var jobVdkImage = - (jobDeployment.getVdkImage() != null) - ? jobDeployment.getVdkImage() + var jobVdkImage = (supportedPythonVersions.isVdkVersionPassedDifferentFromOneSetByPythonVersion(jobDeployment)) + ? supportedPythonVersions.replaceVdkVersionInImage(jobDeployment.getVdkVersion()) : supportedPythonVersions.getVdkImage(jobDeployment.getPythonVersion()); var jobInitContainer = diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployerV2.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployerV2.java index c8f4017962..7fe0ddd936 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployerV2.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployerV2.java @@ -373,10 +373,7 @@ private KubernetesService.CronJob getCronJob( "-c", "cp -r $(python -c \"from distutils.sysconfig import get_python_lib;" + " print(get_python_lib())\") /vdk/. && cp /usr/local/bin/vdk /vdk/."); - var jobVdkImage = - (jobDeployment.getVdkImage() != null) - ? jobDeployment.getVdkImage() - : supportedPythonVersions.getVdkImage(jobDeployment.getPythonVersion()); + var jobVdkImage = supportedPythonVersions.getVdkImage(jobDeployment.getPythonVersion()); var jobInitContainer = KubernetesService.container( "vdk", diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/SupportedPythonVersions.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/SupportedPythonVersions.java index 3775a88ff3..902f50c694 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/SupportedPythonVersions.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/SupportedPythonVersions.java @@ -7,6 +7,7 @@ import java.util.*; +import com.vmware.taurus.service.model.JobDeployment; import org.springframework.beans.factory.annotation.Value; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; @@ -134,4 +135,16 @@ public String getDefaultPythonVersion() { private String getBuilderImage(Map supportedPythonVersion) { return supportedPythonVersion.getOrDefault(BUILDER_IMAGE, dockerRegistryService.builderImage()); } + + public boolean isVdkVersionPassedDifferentFromOneSetByPythonVersion(JobDeployment jobDeployment) { + var passedVdkVersion = jobDeployment.getVdkVersion(); + var vdkVersionSetByPythonVersion = DockerImageName.getTag(getVdkImage(jobDeployment.getPythonVersion())); + + return passedVdkVersion != null && !passedVdkVersion.equals(vdkVersionSetByPythonVersion); + } + + public String replaceVdkVersionInImage(String vdkVersion) { + String defaultVdkImage = getDefaultVdkImage(); + return defaultVdkImage.replace(DockerImageName.getTag(defaultVdkImage), vdkVersion); + } } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java index fe3fa6f8ee..ec6c198b6f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java @@ -59,7 +59,7 @@ public static ExecutionResult getResult(KubernetesService.JobExecution jobExecut return ExecutionResult.builder() .executionStatus(executionStatus) - .vdkImage(podTerminationMessage.getVdkVersion()) + .vdkVersion(podTerminationMessage.getVdkVersion()) .build(); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java index e3c4ff33ec..3982065abe 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java @@ -349,7 +349,7 @@ public Optional updateJobExecu executionStatus, jobExecution.getMainContainerTerminationReason())) .opId(jobExecution.getOpId()) .endTime(jobExecution.getEndTime()) - .vdkImage(executionResult.getVdkImage()) + .vdkVersion(executionResult.getVdkVersion()) .jobVersion(jobExecution.getJobVersion()) .jobPythonVersion(jobExecution.getJobPythonVersion()) .jobSchedule(jobExecution.getJobSchedule()) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/BaseDataJobDeployment.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/BaseDataJobDeployment.java index 9b924ba44f..9a248ca1c9 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/BaseDataJobDeployment.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/BaseDataJobDeployment.java @@ -25,6 +25,8 @@ public abstract class BaseDataJobDeployment { private String pythonVersion; + private String vdkVersion; + private String gitCommitSha; private String schedule; @@ -34,6 +36,4 @@ public abstract class BaseDataJobDeployment { private String lastDeployedBy; private Boolean enabled; - - private String vdkImage; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/DataJobExecution.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/DataJobExecution.java index f8a58e81f9..1fca4a5732 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/DataJobExecution.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/DataJobExecution.java @@ -46,7 +46,7 @@ public class DataJobExecution { private OffsetDateTime endTime; - private String vdkImage; + private String vdkVersion; private String jobVersion; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/ExecutionResult.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/ExecutionResult.java index cf9660e95f..9a6eb483f6 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/ExecutionResult.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/ExecutionResult.java @@ -12,5 +12,5 @@ @Data public class ExecutionResult { private ExecutionStatus executionStatus; - private String vdkImage; + private String vdkVersion; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeployment.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeployment.java index 5c0eb91369..165a601d27 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeployment.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeployment.java @@ -21,7 +21,7 @@ public class JobDeployment { private String gitCommitSha; - private String vdkImage; + private String vdkVersion; private String pythonVersion; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeploymentStatus.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeploymentStatus.java index e9b0fc85e3..30eddd6631 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeploymentStatus.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobDeploymentStatus.java @@ -16,7 +16,9 @@ public class JobDeploymentStatus { private String gitCommitSha; - private String vdkImage; + private String vdkImageName; + + private String vdkVersion; private String pythonVersion; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124124300__add_column_vdk_version_to_desired_data_job_deployment_table.sql b/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124124300__add_column_vdk_version_to_desired_data_job_deployment_table.sql index 5ee2b66172..da0b820f36 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124124300__add_column_vdk_version_to_desired_data_job_deployment_table.sql +++ b/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124124300__add_column_vdk_version_to_desired_data_job_deployment_table.sql @@ -1,2 +1,2 @@ alter table if exists desired_data_job_deployment - add column if not exists vdk_image varchar; + add column if not exists vdk_version varchar; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124140200__add_column_vdk_version_to_actual_data_job_deployment_table.sql b/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124140200__add_column_vdk_version_to_actual_data_job_deployment_table.sql index 27b829a6e9..5a01b6ace7 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124140200__add_column_vdk_version_to_actual_data_job_deployment_table.sql +++ b/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231124140200__add_column_vdk_version_to_actual_data_job_deployment_table.sql @@ -1,2 +1,2 @@ alter table if exists actual_data_job_deployment - add column if not exists vdk_image varchar; + add column if not exists vdk_version varchar; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231130230000__rename_column_vdk_version_to_vdk_image_in_datajob_executions.sql b/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231130230000__rename_column_vdk_version_to_vdk_image_in_datajob_executions.sql deleted file mode 100644 index 0709149e6c..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20231130230000__rename_column_vdk_version_to_vdk_image_in_datajob_executions.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table if exists data_job_execution - rename column vdk_version to vdk_image; diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerReadTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerReadTest.java index bf38a94819..2b37dff893 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerReadTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerReadTest.java @@ -101,7 +101,6 @@ private ActualDataJobDeployment createActualJobDeployment(DataJob dataJob) { deployment.setEnabled(true); deployment.setLastDeployedBy("user"); deployment.setSchedule("sched"); - deployment.setVdkImage("example.com/versatiledatakit/vdk:test-image"); var resources = new DataJobDeploymentResources(); resources.setMemoryLimitMi(1); resources.setMemoryRequestMi(1); diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerTest.java index 155947a05a..4af06fd47b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DataJobDeploymentControllerTest.java @@ -59,24 +59,6 @@ public void testDeploymentPatch_expectDeploymentMerge() { dataJobsDeploymentController.deploymentPatch("teamName", "jobName", "id", newDeployment); var patchedJob = desiredJobDeploymentRepository.findById("jobName").get(); Assertions.assertEquals("jobVersion", patchedJob.getGitCommitSha()); - Assertions.assertEquals("", patchedJob.getVdkImage()); - } - - @Test - @WithMockUser - public void testDeploymentPatch_changeVdkVersion() { - var dataJob = ToModelApiConverter.toDataJob(TestUtils.getDataJob("teamName", "jobName")); - jobsRepository.save(dataJob); - var existingDeployment = getDesiredJobDeployment(dataJob); - desiredJobDeploymentRepository.save(existingDeployment); - - var newDeployment = TestUtils.getDataJobDeployment("deploymentId", "jobVersion"); - newDeployment.setVdkImage("newVdkVersion"); - - dataJobsDeploymentController.deploymentPatch("teamName", "jobName", "id", newDeployment); - - var patchedJob = desiredJobDeploymentRepository.findById("jobName").get(); - Assertions.assertEquals("newVdkVersion", patchedJob.getVdkImage()); } @Test @@ -107,7 +89,6 @@ private DesiredDataJobDeployment getDesiredJobDeployment(DataJob dataJob) { deployment.setEnabled(true); deployment.setLastDeployedBy("user"); deployment.setSchedule("sched"); - deployment.setVdkImage("example.com/versatiledatakit/vdk:test-image"); var resources = new DataJobDeploymentResources(); resources.setMemoryLimitMi(1); resources.setMemoryRequestMi(1); diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceV2Test.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceV2Test.java index 3afc5033f7..9ead470ebf 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceV2Test.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceV2Test.java @@ -98,7 +98,7 @@ public void testFindAllDesiredDataJobDeployments() { public void testFindAllActualDataJobDeployments() { Assertions.assertEquals( - 0, deploymentServiceV2.findAllActualDataJobDeployments().keySet().size()); + 0, deploymentServiceV2.findAllActualDataJobDeployments().keySet().size()); var deployment = new ActualDataJobDeployment(); var dataJob = ToModelApiConverter.toDataJob(TestUtils.getDataJob("teamName", "jobName")); @@ -108,23 +108,7 @@ public void testFindAllActualDataJobDeployments() { actualJobDeploymentRepository.save(deployment); Assertions.assertEquals( - 1, deploymentServiceV2.findAllActualDataJobDeployments().keySet().size()); - } - - @Test - public void testPatchDesiredDeployment_changeVdkVersion_expectMergedDeployment() { - var dataJob = ToModelApiConverter.toDataJob(TestUtils.getDataJob("teamName", "jobName")); - jobsRepository.save(dataJob); - var initialDeployment = new DesiredDataJobDeployment(); - initialDeployment.setDataJob(dataJob); - initialDeployment.setDataJobName(dataJob.getName()); - desiredJobDeploymentRepository.save(initialDeployment); - JobDeployment jobDeployment = generateTestDeployment(); - jobDeployment.setVdkImage("new/test/vdkImage"); - deploymentServiceV2.patchDesiredDbDeployment(dataJob, jobDeployment, "user"); - var savedDeployment = desiredJobDeploymentRepository.findById("jobName").get(); - compareSavedDeploymentWithTestDeployment(jobDeployment, savedDeployment, "user"); - Assertions.assertEquals(jobDeployment.getVdkImage(), savedDeployment.getVdkImage()); + 1, deploymentServiceV2.findAllActualDataJobDeployments().keySet().size()); } @AfterEach diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/SupportedPythonVersionsTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/SupportedPythonVersionsTest.java index bf41250425..7db7df6ff6 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/SupportedPythonVersionsTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/SupportedPythonVersionsTest.java @@ -5,6 +5,7 @@ package com.vmware.taurus.service.deploy; +import com.vmware.taurus.service.model.JobDeployment; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -231,7 +232,7 @@ public void getVdkImage_defaultImage() { supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions); ReflectionTestUtils.setField(supportedPythonVersions, DEFAULT_PYTHON_VERSION, "3.7"); - final String defaultVdkImage = "test_vdk_image_3.7"; + final String defaultVdkImage = "domain:5000/name:test_vdk_image_3.7"; Assertions.assertEquals(defaultVdkImage, supportedPythonVersions.getVdkImage("3.11")); } @@ -240,17 +241,52 @@ public void getVdkImage_defaultImage() { public void getVdkImage_multipleSupportedVersions() { var supportedVersions = generateSupportedPythonVersionsConf(); - final String resultVdkImg = "test_vdk_image_3.8"; + final String resultVdkImg = "domain:5000/name:test_vdk_image_3.7"; ReflectionTestUtils.setField( supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions); Assertions.assertEquals(resultVdkImg, supportedPythonVersions.getVdkImage("3.8")); } + @Test + public void isVdkVersionPassedDifferentFromOneSetInPythonVersion_shouldReturnTrue() { + var supportedVersions = generateSupportedPythonVersionsConf(); + ReflectionTestUtils.setField( + supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions); + JobDeployment testDeployment = new JobDeployment(); + testDeployment.setPythonVersion("3.8"); + testDeployment.setVdkVersion("test_vdk_image_3.7"); + + Assertions.assertTrue(supportedPythonVersions.isVdkVersionPassedDifferentFromOneSetByPythonVersion(testDeployment)); + } + + @Test + public void isVdkVersionPassedDifferentFromOneSetInPythonVersion_shouldReturnFalse() { + var supportedVersions = generateSupportedPythonVersionsConf(); + ReflectionTestUtils.setField( + supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions); + JobDeployment testDeployment = new JobDeployment(); + testDeployment.setPythonVersion("3.8"); + testDeployment.setVdkVersion("test_vdk_image_3.8"); + + Assertions.assertFalse(supportedPythonVersions.isVdkVersionPassedDifferentFromOneSetByPythonVersion(testDeployment)); + } + + @Test + public void replaceVdkVersionInImage_replaceTheImage() { + var supportedVersions = generateSupportedPythonVersionsConf(); + ReflectionTestUtils.setField( + supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions); + ReflectionTestUtils.setField(supportedPythonVersions, DEFAULT_PYTHON_VERSION, "3.8"); + String resImage = "domain:5000/name:replaced_vdk_image"; + + Assertions.assertEquals(resImage, supportedPythonVersions.replaceVdkVersionInImage("replaced_vdk_image")); + } + private static Map> generateSupportedPythonVersionsConf() { return Map.of( - "3.7", Map.of(BASE_IMAGE, "python:3.7-slim", VDK_IMAGE, "test_vdk_image_3.7"), - "3.8", Map.of(BASE_IMAGE, "python:3.8-slim", VDK_IMAGE, "test_vdk_image_3.8"), + "3.7", Map.of(BASE_IMAGE, "python:3.7-slim", VDK_IMAGE, "domain:5000/name:test_vdk_image_3.7"), + "3.8", Map.of(BASE_IMAGE, "python:3.8-slim", VDK_IMAGE, "domain:5000/name:test_vdk_image_3.8"), "3.9", Map.of(BASE_IMAGE, "python:3.9-slim", VDK_IMAGE, "test_vdk_image_3.9")); } }