From f254eb00173191b736564ce2cf740c62c55a5acc Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Fri, 13 May 2022 18:15:34 +0300 Subject: [PATCH 1/9] control-service: Allow for jobs with no schedule to be deployed Currently, if a job is deployed with no schedule_cron, the deployment will fail. After this change, jobs with no schedule will be deployed successfully, their cronjobs will have a default schedule "0 0 30 2 *" - meaning Feb 30, or never - but the job metadata will have an empty string for a schedule, so that "vdk list" will not print a schedule for those jobs (but jobs with their schedule explicitly set to "0 0 30 2 *" will still have it show). Testing done: TBD Signed-off-by: Gabriel Georgiev --- .../com/vmware/taurus/service/deploy/JobImageDeployer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java index 337556073a..700d52152f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java @@ -167,7 +167,9 @@ private void updateCronJob(DataJob dataJob, JobDeployment jobDeployment, String lastDeployedBy) throws ApiException { log.debug("Deploy cron job for data job {}", dataJob); - String schedule = dataJob.getJobConfig().getSchedule(); + + // Schedule defaults to Feb 30 (i.e. never) if no schedule has been given. + String schedule = (dataJob.getJobConfig().getSchedule().isEmpty() || dataJob.getJobConfig().getSchedule() == null) ? "0 0 30 2 *" : dataJob.getJobConfig().getSchedule(); var jobName = dataJob.getName(); var volume = KubernetesService.volume(VOLUME_NAME); @@ -286,6 +288,9 @@ private Map getJobLabels(DataJob dataJob, JobDeployment jobDeplo private Map getJobAnnotations(DataJob dataJob, String deployedBy){ Map jobPodAnnotations = new HashMap<>(); + // HERE MAYBE? + // String schedule = dataJob.getJobConfig().getSchedule() == null ? "" : dataJob.getJobConfig().getSchedule(); + // jobPodAnnotations.put(JobAnnotation.SCHEDULE.getValue(), schedule); jobPodAnnotations.put(JobAnnotation.SCHEDULE.getValue(), dataJob.getJobConfig().getSchedule()); jobPodAnnotations.put(JobAnnotation.EXECUTION_TYPE.getValue(), "scheduled"); jobPodAnnotations.put(JobAnnotation.DEPLOYED_BY.getValue(), deployedBy); From 78758062dd88a37bca09ee19277646fc6d494498 Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Fri, 13 May 2022 18:41:16 +0300 Subject: [PATCH 2/9] extra Signed-off-by: Gabriel Georgiev --- .../vmware/taurus/service/deploy/JobImageDeployer.java | 9 ++++----- .../com/vmware/taurus/service/model/JobAnnotation.java | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java index 700d52152f..f3c2bff23f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import javax.print.DocFlavor; import java.time.OffsetDateTime; import java.util.*; @@ -169,7 +170,7 @@ private void updateCronJob(DataJob dataJob, log.debug("Deploy cron job for data job {}", dataJob); // Schedule defaults to Feb 30 (i.e. never) if no schedule has been given. - String schedule = (dataJob.getJobConfig().getSchedule().isEmpty() || dataJob.getJobConfig().getSchedule() == null) ? "0 0 30 2 *" : dataJob.getJobConfig().getSchedule(); + String schedule = StringUtils.isEmpty(dataJob.getJobConfig().getSchedule()) ? "0 0 30 2 *" : dataJob.getJobConfig().getSchedule(); var jobName = dataJob.getName(); var volume = KubernetesService.volume(VOLUME_NAME); @@ -288,14 +289,12 @@ private Map getJobLabels(DataJob dataJob, JobDeployment jobDeplo private Map getJobAnnotations(DataJob dataJob, String deployedBy){ Map jobPodAnnotations = new HashMap<>(); - // HERE MAYBE? - // String schedule = dataJob.getJobConfig().getSchedule() == null ? "" : dataJob.getJobConfig().getSchedule(); - // jobPodAnnotations.put(JobAnnotation.SCHEDULE.getValue(), schedule); jobPodAnnotations.put(JobAnnotation.SCHEDULE.getValue(), dataJob.getJobConfig().getSchedule()); - jobPodAnnotations.put(JobAnnotation.EXECUTION_TYPE.getValue(), "scheduled"); + jobPodAnnotations.put(JobAnnotation.EXECUTION_TYPE.getValue(), (StringUtils.isEmpty(dataJob.getJobConfig().getSchedule()) ? "unscheduled" : "scheduled")); jobPodAnnotations.put(JobAnnotation.DEPLOYED_BY.getValue(), deployedBy); jobPodAnnotations.put(JobAnnotation.DEPLOYED_DATE.getValue(), OffsetDateTime.now().toString()); jobPodAnnotations.put(JobAnnotation.STARTED_BY.getValue(), "scheduled/runtime"); //TODO are those valid? + jobPodAnnotations.put(JobAnnotation.UNSCHEDULED.getValue(), (StringUtils.isEmpty(dataJob.getJobConfig().getSchedule()) ? "true" : "false")); return jobPodAnnotations; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobAnnotation.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobAnnotation.java index 7b6815ad16..0b7541beee 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobAnnotation.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobAnnotation.java @@ -14,7 +14,8 @@ public enum JobAnnotation { DEPLOYED_DATE("deployed-date"), DEPLOYED_BY("deployed-by"), EXECUTION_TYPE("execution-type"), - OP_ID("op-id"); + OP_ID("op-id"), + UNSCHEDULED("unscheduled"); @Getter private String value; From 4b4e4520f68b5a7c6a5229301a3eb04948b4ab83 Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Fri, 13 May 2022 18:42:01 +0300 Subject: [PATCH 3/9] dropped unused import Signed-off-by: Gabriel Georgiev --- .../java/com/vmware/taurus/service/deploy/JobImageDeployer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java index f3c2bff23f..20b14f2ec2 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.print.DocFlavor; import java.time.OffsetDateTime; import java.util.*; From 578247e7bc1ceb3df775d01916807238d1abde14 Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Fri, 13 May 2022 19:26:13 +0300 Subject: [PATCH 4/9] fix jobannotations test Signed-off-by: Gabriel Georgiev --- .../taurus/datajobs/DataJobsControllerTest.java | 13 +++++++++++++ .../service/model/K8SAnnotationsLabelsTest.java | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java index b71bad1fdd..2857a38c69 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java @@ -52,6 +52,19 @@ public void testDataJobCreate() { assertEquals("http://test.com/data-jobs/for-team/team/jobs/test-job", location); } + @Test + public void testDataJobCreateUnscheduledJob() { + when(jobsService.createJob(any())).thenReturn(JobOperationResult.builder() + .completed(true) + .build()); + UriBuilder builder = UriComponentsBuilder.fromHttpUrl("http://test.com/"); + var testInst = new DataJobsController(jobsService, null, null, null, () -> builder); + DataJob dataJob = newJob("test-job", "team", ""); + var response = testInst.dataJobCreate("team", dataJob, ""); + var location = response.getHeaders().get(HttpHeaders.LOCATION).get(0); + assertEquals("http://test.com/data-jobs/for-team/team/jobs/test-job", location); + } + @Test public void testDataJobCreateNameValidity() { when(jobsService.createJob(any())).thenReturn(JobOperationResult.builder() diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/model/K8SAnnotationsLabelsTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/model/K8SAnnotationsLabelsTest.java index 0856f9bd0c..c71343a803 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/model/K8SAnnotationsLabelsTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/model/K8SAnnotationsLabelsTest.java @@ -26,6 +26,7 @@ public void testAnnotations() { var deployedBy = JobAnnotation.DEPLOYED_BY.getValue(); var executionType = JobAnnotation.EXECUTION_TYPE.getValue(); var opId = JobAnnotation.OP_ID.getValue(); + var unscheduled = JobAnnotation.UNSCHEDULED.getValue(); Assertions.assertEquals("com.vmware.taurus/schedule", schedule, msg); Assertions.assertEquals("com.vmware.taurus/started-by", startedBy, msg); @@ -33,8 +34,9 @@ public void testAnnotations() { Assertions.assertEquals("com.vmware.taurus/deployed-by", deployedBy, msg); Assertions.assertEquals("com.vmware.taurus/execution-type", executionType, msg); Assertions.assertEquals("com.vmware.taurus/op-id", opId, msg); + Assertions.assertEquals("com.vmware.taurus/unscheduled", unscheduled, msg); - Assertions.assertEquals(6, JobAnnotation.values().length, msg); + Assertions.assertEquals(7, JobAnnotation.values().length, msg); } @Test From 15d0c3edf5897af008145bd9292c422ac39532bb Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Mon, 16 May 2022 16:51:03 +0300 Subject: [PATCH 5/9] unit test Signed-off-by: Gabriel Georgiev --- .../resources/application-debug.properties | 71 +++++++++++++++++++ .../datajobs/DataJobsControllerTest.java | 13 ---- .../service/deploy/JobImageDeployerTest.java | 24 +++++++ 3 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties b/projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties new file mode 100644 index 0000000000..33a7f80f7b --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties @@ -0,0 +1,71 @@ +datajobs.kadmin_user= +datajobs.kadmin_password= + +logging.level.com.vmware.taurus=DEBUG + +datajobs.aws.region= +datajobs.aws.accessKeyId= +datajobs.aws.secretAccessKey= + +# Path to an ini config file that contains vdk runtime options +# src/main/resources/vdk_options.ini can be used for testing +datajobs.vdk_options_ini= +datajobs.vdk.image=registry.hub.docker.com/versatiledatakit/quickstart-vdk:release + +datajobs.deployment.k8s.kubeconfig=${DEPLOYMENT_K8S_KUBECONFIG} +datajobs.deployment.k8s.namespace=${DEPLOYMENT_K8S_NAMESPACE} +datajobs.control.k8s.k8sSupportsV1CronJob=false + +datajobs.control.k8s.kubeconfig=${CONTROL_K8S_KUBECONFIG} +datajobs.control.k8s.namespace=${CONTROL_K8S_NAMESPACE} + +datajobs.docker.repositoryUrl=${DOCKER_REGISTRY_URL} +datajobs.docker.registryType=generic + +# Credentials for generic registry type like Harbor or Dockerhub. +datajobs.docker.registryUsername=${DOCKER_REGISTRY_USERNAME} +datajobs.docker.registryPassword=${DOCKER_REGISTRY_PASSWORD} +datajobs.docker.registrySecret=${DOCKER_REGISTRY_SECRET:} + +#WebHook settings for the integration tests +integrationTest.mockedWebHookServerHost=localhost +integrationTest.mockedWebHookServerPort=5878 +datajobs.authorization.webhook.endpoint=https://httpbin.org/post +# datajobs.post.create.webhook.endpoint=http://localhost:5878 +# datajobs.post.create.webhook.internal.errors.retries=3 +# datajobs.post.delete.webhook.endpoint=http://localhost:5878 +# datajobs.post.delete.webhook.internal.errors.retries=3 + +featureflag.security.enabled=false +spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://test + +# Required +# Please specify valid values for all of the properties below + +# By default test tries to create namespace dynamically - in this case leave them empty. +# If in your kubernetes that is not possible, set a fixed namespace here +integrationTest.dataJobsNamespace=${DEPLOYMENT_K8S_NAMESPACE} +integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE} + +datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 +datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} +datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest + +datajobs.git.url=${GIT_URL} +datajobs.git.username=${GIT_USERNAME} +datajobs.git.password=${GIT_PASSWORD} +datajobs.git.branch=main + +# For local run, create personal access token in github with read and write access (or use your user and password) +# and export those environmental variables in the terminal or in your IDE depending on how you run the tests . +datajobs.git.read.write.username=${GIT_USERNAME_READ_WRITE} +datajobs.git.read.write.password=${GIT_PASSWORD_READ_WRITE} + +datajobs.executions.logsUrl.template=https://log-insight-base-url/li/query/stream?query=%C2%A7%C2%A7%C2%A7AND%C\ +2%A7%C2%A7%C2%A7%C2%A7{{start_time}}%C2%A7{{end_time}}%C2%A7true%C2%A7COUNT%C2%A7*%C2%A7\ +timestamp%C2%A7pageSortPreference:%7B%22sortBy%22%3A%22-{{job_name}}-{{op_id}}-\ +ingest_timestamp%22%2C%22sortOrder%22%3A%22DESC%22%7D%C2%A7\ +alertDefList:%5B%5D%C2%A7partitions:%C2%A7%C2%A7text:CONTAINS:{{execution_id}}* +datajobs.executions.logsUrl.dateFormat=unix + +featureflag.authorization.enabled=false diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java index 2857a38c69..b71bad1fdd 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/DataJobsControllerTest.java @@ -52,19 +52,6 @@ public void testDataJobCreate() { assertEquals("http://test.com/data-jobs/for-team/team/jobs/test-job", location); } - @Test - public void testDataJobCreateUnscheduledJob() { - when(jobsService.createJob(any())).thenReturn(JobOperationResult.builder() - .completed(true) - .build()); - UriBuilder builder = UriComponentsBuilder.fromHttpUrl("http://test.com/"); - var testInst = new DataJobsController(jobsService, null, null, null, () -> builder); - DataJob dataJob = newJob("test-job", "team", ""); - var response = testInst.dataJobCreate("team", dataJob, ""); - var location = response.getHeaders().get(HttpHeaders.LOCATION).get(0); - assertEquals("http://test.com/data-jobs/for-team/team/jobs/test-job", location); - } - @Test public void testDataJobCreateNameValidity() { when(jobsService.createJob(any())).thenReturn(JobOperationResult.builder() diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageDeployerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageDeployerTest.java index 6a34c30945..a3429a8ba8 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageDeployerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageDeployerTest.java @@ -146,4 +146,28 @@ public void testJobContainerName() throws ApiException { var jobContainer = containerCaptor.getValue(); Assertions.assertEquals(testDataJob.getName(), jobContainer.getName()); } + + @Test + public void testJobNoSchedule() throws ApiException { + JobConfig jobConfig = new JobConfig(); + jobConfig.setSchedule(""); + testDataJob = new DataJob(); + testDataJob.setName(TEST_JOB_NAME); + testDataJob.setJobConfig(jobConfig); + + var jobDeployment = new JobDeployment(); + jobDeployment.setDataJobName(TEST_JOB_NAME); + jobDeployment.setGitCommitSha("test-commit"); + jobDeployment.setEnabled(true); + jobDeployment.setImageName("image"); + + jobImageDeployer.scheduleJob(testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME); + + ArgumentCaptor scheduleCaptor = ArgumentCaptor.forClass(String.class); + verify(kubernetesService).createCronJob(anyString(), anyString(), anyMap(), scheduleCaptor.capture(), anyBoolean(), + anyList(), any(KubernetesService.Resources.class), any(KubernetesService.Resources.class), + any(V1Container.class), any(V1Container.class), anyList(), anyMap(), anyMap(), anyMap(), anyMap(), anyList()); + Assertions.assertEquals("0 0 30 2 *", scheduleCaptor.getValue()); + Assertions.assertEquals("", testDataJob.getJobConfig().getSchedule()); + } } From be95e1b5a22e3e675539e0716eae5e9a6fdd6cb0 Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Mon, 16 May 2022 16:51:52 +0300 Subject: [PATCH 6/9] drop application-debug.properties from pr Signed-off-by: Gabriel Georgiev --- .../resources/application-debug.properties | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties b/projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties deleted file mode 100644 index 33a7f80f7b..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/application-debug.properties +++ /dev/null @@ -1,71 +0,0 @@ -datajobs.kadmin_user= -datajobs.kadmin_password= - -logging.level.com.vmware.taurus=DEBUG - -datajobs.aws.region= -datajobs.aws.accessKeyId= -datajobs.aws.secretAccessKey= - -# Path to an ini config file that contains vdk runtime options -# src/main/resources/vdk_options.ini can be used for testing -datajobs.vdk_options_ini= -datajobs.vdk.image=registry.hub.docker.com/versatiledatakit/quickstart-vdk:release - -datajobs.deployment.k8s.kubeconfig=${DEPLOYMENT_K8S_KUBECONFIG} -datajobs.deployment.k8s.namespace=${DEPLOYMENT_K8S_NAMESPACE} -datajobs.control.k8s.k8sSupportsV1CronJob=false - -datajobs.control.k8s.kubeconfig=${CONTROL_K8S_KUBECONFIG} -datajobs.control.k8s.namespace=${CONTROL_K8S_NAMESPACE} - -datajobs.docker.repositoryUrl=${DOCKER_REGISTRY_URL} -datajobs.docker.registryType=generic - -# Credentials for generic registry type like Harbor or Dockerhub. -datajobs.docker.registryUsername=${DOCKER_REGISTRY_USERNAME} -datajobs.docker.registryPassword=${DOCKER_REGISTRY_PASSWORD} -datajobs.docker.registrySecret=${DOCKER_REGISTRY_SECRET:} - -#WebHook settings for the integration tests -integrationTest.mockedWebHookServerHost=localhost -integrationTest.mockedWebHookServerPort=5878 -datajobs.authorization.webhook.endpoint=https://httpbin.org/post -# datajobs.post.create.webhook.endpoint=http://localhost:5878 -# datajobs.post.create.webhook.internal.errors.retries=3 -# datajobs.post.delete.webhook.endpoint=http://localhost:5878 -# datajobs.post.delete.webhook.internal.errors.retries=3 - -featureflag.security.enabled=false -spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://test - -# Required -# Please specify valid values for all of the properties below - -# By default test tries to create namespace dynamically - in this case leave them empty. -# If in your kubernetes that is not possible, set a fixed namespace here -integrationTest.dataJobsNamespace=${DEPLOYMENT_K8S_NAMESPACE} -integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE} - -datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 -datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} -datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest - -datajobs.git.url=${GIT_URL} -datajobs.git.username=${GIT_USERNAME} -datajobs.git.password=${GIT_PASSWORD} -datajobs.git.branch=main - -# For local run, create personal access token in github with read and write access (or use your user and password) -# and export those environmental variables in the terminal or in your IDE depending on how you run the tests . -datajobs.git.read.write.username=${GIT_USERNAME_READ_WRITE} -datajobs.git.read.write.password=${GIT_PASSWORD_READ_WRITE} - -datajobs.executions.logsUrl.template=https://log-insight-base-url/li/query/stream?query=%C2%A7%C2%A7%C2%A7AND%C\ -2%A7%C2%A7%C2%A7%C2%A7{{start_time}}%C2%A7{{end_time}}%C2%A7true%C2%A7COUNT%C2%A7*%C2%A7\ -timestamp%C2%A7pageSortPreference:%7B%22sortBy%22%3A%22-{{job_name}}-{{op_id}}-\ -ingest_timestamp%22%2C%22sortOrder%22%3A%22DESC%22%7D%C2%A7\ -alertDefList:%5B%5D%C2%A7partitions:%C2%A7%C2%A7text:CONTAINS:{{execution_id}}* -datajobs.executions.logsUrl.dateFormat=unix - -featureflag.authorization.enabled=false From 266695c35d45b5232778e42b89a951d16b398fda Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Mon, 16 May 2022 18:51:36 +0300 Subject: [PATCH 7/9] drop unscheduled as an annotation Signed-off-by: Gabriel Georgiev --- .../java/com/vmware/taurus/service/deploy/JobImageDeployer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java index 20b14f2ec2..318eecde0b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java @@ -289,7 +289,7 @@ private Map getJobLabels(DataJob dataJob, JobDeployment jobDeplo private Map getJobAnnotations(DataJob dataJob, String deployedBy){ Map jobPodAnnotations = new HashMap<>(); jobPodAnnotations.put(JobAnnotation.SCHEDULE.getValue(), dataJob.getJobConfig().getSchedule()); - jobPodAnnotations.put(JobAnnotation.EXECUTION_TYPE.getValue(), (StringUtils.isEmpty(dataJob.getJobConfig().getSchedule()) ? "unscheduled" : "scheduled")); + jobPodAnnotations.put(JobAnnotation.EXECUTION_TYPE.getValue(), "scheduled"); jobPodAnnotations.put(JobAnnotation.DEPLOYED_BY.getValue(), deployedBy); jobPodAnnotations.put(JobAnnotation.DEPLOYED_DATE.getValue(), OffsetDateTime.now().toString()); jobPodAnnotations.put(JobAnnotation.STARTED_BY.getValue(), "scheduled/runtime"); //TODO are those valid? From 92c05e4a0a7539f30e834af25491398136686996 Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Tue, 17 May 2022 09:10:29 +0300 Subject: [PATCH 8/9] fix failing test Signed-off-by: Gabriel Georgiev --- .../java/com/vmware/taurus/service/JobImageDeployerTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java index 9ad596bd1e..668b963e8e 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java @@ -98,7 +98,7 @@ public void testScheduleLabelsAnnotations() { var annotations = annotationCaptor.getValue(); //check everything was as expected Assertions.assertEquals(3, labels.size(), "Expecting three labels"); - Assertions.assertEquals(5, annotations.size(), "Expecting five annotations"); + Assertions.assertEquals(6, annotations.size(), "Expecting five annotations"); var jobName = labels.get(JobLabel.NAME.getValue()); var jobVersion = labels.get(JobLabel.VERSION.getValue()); @@ -113,6 +113,7 @@ public void testScheduleLabelsAnnotations() { var deployedDate = annotations.get(JobAnnotation.DEPLOYED_DATE.getValue()); var startedBy = annotations.get(JobAnnotation.STARTED_BY.getValue()); var executionType = annotations.get(JobAnnotation.EXECUTION_TYPE.getValue()); + var unscheduled = annotations.get(JobAnnotation.UNSCHEDULED.getValue()); Assertions.assertEquals("schedule string", schedule); Assertions.assertEquals("lastDeployedBy", deployedBy); @@ -120,6 +121,7 @@ public void testScheduleLabelsAnnotations() { "Testing if date is the same, we cannot be precise down to the millisecond."); Assertions.assertEquals("scheduled/runtime", startedBy); Assertions.assertEquals("scheduled", executionType); + Assertions.assertEquals("false", unscheduled); } catch (ApiException e) { e.printStackTrace(); From 49d7998e9d587841906c6436332b7f2afe3c0ca3 Mon Sep 17 00:00:00 2001 From: Gabriel Georgiev Date: Tue, 17 May 2022 10:20:29 +0300 Subject: [PATCH 9/9] fix assert msg Signed-off-by: Gabriel Georgiev --- .../java/com/vmware/taurus/service/JobImageDeployerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java index 668b963e8e..1588405e04 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/JobImageDeployerTest.java @@ -98,7 +98,7 @@ public void testScheduleLabelsAnnotations() { var annotations = annotationCaptor.getValue(); //check everything was as expected Assertions.assertEquals(3, labels.size(), "Expecting three labels"); - Assertions.assertEquals(6, annotations.size(), "Expecting five annotations"); + Assertions.assertEquals(6, annotations.size(), "Expecting six annotations"); var jobName = labels.get(JobLabel.NAME.getValue()); var jobVersion = labels.get(JobLabel.VERSION.getValue());