From 9ce88afa663b2a78d607f9746c97f27f8db9500f Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 9 May 2023 11:09:04 +0100 Subject: [PATCH 1/5] control-service: fix oom tests Signed-off-by: murphp15 --- .../it/DataJobInitContainerOOMIT.java | 22 ++++++---- .../datajobs/it/common/JobExecutionUtil.java | 6 +-- .../resources/fast_failing_cron_job.yaml | 41 +++++++++++++++++++ .../taurus/service/KubernetesService.java | 18 +------- 4 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java index 31d6638515..e68ce1995b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java @@ -12,15 +12,21 @@ import com.vmware.taurus.datajobs.it.common.JobExecutionUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.TestPropertySource; +import static com.vmware.taurus.datajobs.it.common.JobExecutionUtil.*; + @Slf4j @TestPropertySource( properties = { "datajobs.deployment.initContainer.resources.requests.memory=6Mi", "datajobs.deployment.initContainer.resources.limits.memory=6Mi", + // This is a standard cron job template except restartPolicy is set to never so that when a job runs out of memory it is + // not retied but instead reports more quickly that it is a platform error + "datajobs.control.k8s.data.job.template.file=fast_failing_cron_job.yaml" }) @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, @@ -30,7 +36,7 @@ public class DataJobInitContainerOOMIT extends BaseIT { @RegisterExtension static DataJobDeploymentExtension dataJobDeploymentExtension = new DataJobDeploymentExtension(); - // @Test + @Test public void testDataJob_causesOOM_shouldCompleteWithUserError( String jobName, String teamName, String username, String deploymentId) throws Exception { // manually start job execution @@ -40,13 +46,11 @@ public void testDataJob_causesOOM_shouldCompleteWithUserError( String executionId = executeDataJobResult.getRight(); // Check the data job execution status - JobExecutionUtil.checkDataJobExecutionStatus( - executionId, - DataJobExecution.StatusEnum.PLATFORM_ERROR, - opId, - jobName, - teamName, - username, - mockMvc); + testDataJobExecutionRead( + executionId, DataJobExecution.StatusEnum.PLATFORM_ERROR, opId, jobName, teamName, username, mockMvc); + testDataJobExecutionList( + executionId, DataJobExecution.StatusEnum.PLATFORM_ERROR, opId, jobName, teamName, username, mockMvc); + testDataJobDeploymentExecutionList( + executionId, DataJobExecution.StatusEnum.PLATFORM_ERROR, opId, jobName, teamName, username, mockMvc); } } diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java index a74f8bfcd5..df6cdad06d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java @@ -150,7 +150,7 @@ public static ImmutablePair executeDataJob( return ImmutablePair.of(opId, executionId); } - private static void testDataJobExecutionRead( + public static void testDataJobExecutionRead( String executionId, com.vmware.taurus.controlplane.model.data.DataJobExecution.StatusEnum executionStatus, String opId, @@ -198,7 +198,7 @@ private static void testDataJobExecutionRead( executionId, executionStatus, opId, dataJobExecution[0], jobName, username); } - private static void testDataJobExecutionList( + public static void testDataJobExecutionList( String executionId, com.vmware.taurus.controlplane.model.data.DataJobExecution.StatusEnum executionStatus, String opId, @@ -232,7 +232,7 @@ private static void testDataJobExecutionList( executionId, executionStatus, opId, dataJobExecutions.get(0), jobName, username); } - private static void testDataJobDeploymentExecutionList( + public static void testDataJobDeploymentExecutionList( String executionId, com.vmware.taurus.controlplane.model.data.DataJobExecution.StatusEnum executionStatus, String opId, diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml new file mode 100644 index 0000000000..7302d8a9ea --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml @@ -0,0 +1,41 @@ +# Copyright 2021-2023 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: # merged with additional annotations from TPCS + name: cronjob-template-name # overridden by TPCS +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 2 + schedule: "*/10 * * * *" # overridden by TPCS + startingDeadlineSeconds: 1800 + successfulJobsHistoryLimit: 1 + suspend: false # overridden by TPCS + jobTemplate: + metadata: + annotations: # merged with additional annotations from TPCS + labels: # merged with additional labels from TPCS + spec: + activeDeadlineSeconds: 43200 + backoffLimit: 3 + template: + metadata: + labels: # merged with additional labels from TPCS + spec: + containers: # overridden by TPCS + - command: + - /bin/sh + - -c + - date; echo '************** Cronjob Template ******************' + name: cronjob-template-container-name + image: busybox + imagePullPolicy: IfNotPresent + restartPolicy: Never + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + automountServiceAccountToken: false + ttlSecondsAfterFinished: 600 diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 77f9fa1d31..a87a0e3dc2 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -332,16 +332,9 @@ private V1beta1CronJob loadConfigurableV1beta1CronjobTemplate() { return null; } - // Check whether the configurable datajob template file exists. - File datajobTemplateFile = new File(datajobTemplateFileLocation); - if (!datajobTemplateFile.isFile()) { - log.warn("Datajob template location '{}' is not a file.", datajobTemplateFileLocation); - return null; - } - try { // Load the configurable datajob template file. - return loadV1beta1CronjobTemplate(datajobTemplateFile); + return loadV1beta1CronjobTemplate(new ClassPathResource(datajobTemplateFileLocation).getFile()); } catch (Exception e) { log.error("Error while loading the datajob template file.", e); return null; @@ -355,16 +348,9 @@ private V1CronJob loadConfigurableV1CronjobTemplate() { return null; } - // Check whether the configurable datajob template file exists. - File datajobTemplateFile = new File(datajobTemplateFileLocation); - if (!datajobTemplateFile.isFile()) { - log.warn("Datajob template location '{}' is not a file.", datajobTemplateFileLocation); - return null; - } - try { // Load the configurable datajob template file. - return loadV1CronjobTemplate(datajobTemplateFile); + return loadV1CronjobTemplate(new ClassPathResource(datajobTemplateFileLocation).getFile()); } catch (Exception e) { log.error("Error while loading the datajob template file.", e); return null; From ba02b88bd415fecad484d602c8b4f2cd1389c737 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 9 May 2023 10:09:29 +0000 Subject: [PATCH 2/5] Google Java Format --- .../it/DataJobInitContainerOOMIT.java | 31 +++++++++++++++---- .../taurus/service/KubernetesService.java | 3 +- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java index e68ce1995b..86d1e52566 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java @@ -24,9 +24,10 @@ properties = { "datajobs.deployment.initContainer.resources.requests.memory=6Mi", "datajobs.deployment.initContainer.resources.limits.memory=6Mi", - // This is a standard cron job template except restartPolicy is set to never so that when a job runs out of memory it is - // not retied but instead reports more quickly that it is a platform error - "datajobs.control.k8s.data.job.template.file=fast_failing_cron_job.yaml" + // This is a standard cron job template except restartPolicy is set to never so that when a + // job runs out of memory it is + // not retied but instead reports more quickly that it is a platform error + "datajobs.control.k8s.data.job.template.file=fast_failing_cron_job.yaml" }) @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, @@ -47,10 +48,28 @@ public void testDataJob_causesOOM_shouldCompleteWithUserError( // Check the data job execution status testDataJobExecutionRead( - executionId, DataJobExecution.StatusEnum.PLATFORM_ERROR, opId, jobName, teamName, username, mockMvc); + executionId, + DataJobExecution.StatusEnum.PLATFORM_ERROR, + opId, + jobName, + teamName, + username, + mockMvc); testDataJobExecutionList( - executionId, DataJobExecution.StatusEnum.PLATFORM_ERROR, opId, jobName, teamName, username, mockMvc); + executionId, + DataJobExecution.StatusEnum.PLATFORM_ERROR, + opId, + jobName, + teamName, + username, + mockMvc); testDataJobDeploymentExecutionList( - executionId, DataJobExecution.StatusEnum.PLATFORM_ERROR, opId, jobName, teamName, username, mockMvc); + executionId, + DataJobExecution.StatusEnum.PLATFORM_ERROR, + opId, + jobName, + teamName, + username, + mockMvc); } } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index a87a0e3dc2..3f17e79a7a 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -334,7 +334,8 @@ private V1beta1CronJob loadConfigurableV1beta1CronjobTemplate() { try { // Load the configurable datajob template file. - return loadV1beta1CronjobTemplate(new ClassPathResource(datajobTemplateFileLocation).getFile()); + return loadV1beta1CronjobTemplate( + new ClassPathResource(datajobTemplateFileLocation).getFile()); } catch (Exception e) { log.error("Error while loading the datajob template file.", e); return null; From d6dcf2aaf05d47c8f385d7117b0be0e71bc1b701 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 11 May 2023 10:06:50 +0100 Subject: [PATCH 3/5] v1 to v1Beta1 --- .../src/integration-test/resources/fast_failing_cron_job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml index 7302d8a9ea..e497796e69 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/fast_failing_cron_job.yaml @@ -1,7 +1,7 @@ # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 +apiVersion: batch/v1beta1 kind: CronJob metadata: annotations: # merged with additional annotations from TPCS From f4590b72981986647505db83eb2af81131ecf129 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 11 May 2023 13:38:36 +0100 Subject: [PATCH 4/5] control-service: report tests correctly Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 31ebc9fecd..81a67e58b0 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -93,10 +93,10 @@ control_service_integration_test: artifacts: when: always paths: - - ./projects/control-service/projects/*/build/reports/** + - ./projects/control-service/projects/pipelines_control_service/build/reports/tests/integrationTest/index.html expire_in: 1 week reports: - junit: ./projects/control-service/projects/**/build/test-results/integrationTest/TEST-*.xml + junit: ./projects/control-service/projects/pipelines_control_service/build/reports/tests/integrationTest/index.html only: refs: - external_pull_requests From 66bfe013b64021bb3e3b6b43baa7f2fdc7193494 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 11 May 2023 14:15:06 +0100 Subject: [PATCH 5/5] control-service: revert Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 81a67e58b0..31ebc9fecd 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -93,10 +93,10 @@ control_service_integration_test: artifacts: when: always paths: - - ./projects/control-service/projects/pipelines_control_service/build/reports/tests/integrationTest/index.html + - ./projects/control-service/projects/*/build/reports/** expire_in: 1 week reports: - junit: ./projects/control-service/projects/pipelines_control_service/build/reports/tests/integrationTest/index.html + junit: ./projects/control-service/projects/**/build/test-results/integrationTest/TEST-*.xml only: refs: - external_pull_requests