diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java new file mode 100644 index 0000000000..449dbe1089 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobInitContainerOOMIT.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021-2023 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.controlplane.model.data.DataJobExecution; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.datajobs.it.common.DataJobDeploymentExtension; +import com.vmware.taurus.datajobs.it.common.JobExecutionUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +@Slf4j +@TestPropertySource( + properties = { + "datajobs.deployment.initContainer.resources.requests.memory=6Mi", + "datajobs.deployment.initContainer.resources.limits.memory=6Mi", + }) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class DataJobInitContainerOOMIT extends BaseIT { + + @RegisterExtension + static DataJobDeploymentExtension dataJobDeploymentExtension = new DataJobDeploymentExtension(); + + @Test + public void testDataJob_causesOOM_shouldCompleteWithUserError( + String jobName, String teamName, String username, String deploymentId) throws Exception { + // manually start job execution + ImmutablePair executeDataJobResult = + JobExecutionUtil.executeDataJob(jobName, teamName, username, deploymentId, mockMvc); + String opId = executeDataJobResult.getLeft(); + String executionId = executeDataJobResult.getRight(); + + // Check the data job execution status + JobExecutionUtil.checkDataJobExecutionStatus( + executionId, + DataJobExecution.StatusEnum.PLATFORM_ERROR, + opId, + jobName, + teamName, + username, + mockMvc); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobOOMIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobMainContainerOOMIT.java similarity index 96% rename from projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobOOMIT.java rename to projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobMainContainerOOMIT.java index 523b77cd1c..62cd5d8c6c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobOOMIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobMainContainerOOMIT.java @@ -20,7 +20,7 @@ @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = ControlplaneApplication.class) -public class DataJobOOMIT extends BaseIT { +public class DataJobMainContainerOOMIT extends BaseIT { @RegisterExtension static DataJobDeploymentExtension dataJobDeploymentExtension = diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index b3ae192734..f5ba569711 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -40,6 +40,7 @@ import lombok.*; import net.javacrumbs.shedlock.spring.annotation.EnableSchedulerLock; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.springframework.beans.factory.InitializingBean; @@ -58,6 +59,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import static java.util.function.Predicate.not; @@ -145,7 +147,7 @@ public static class JobExecution { String executionId; String executionType; String jobName; - String podTerminationMessage; + String mainContainerTerminationMessage; String jobTerminationReason; Boolean succeeded; String opId; @@ -159,7 +161,8 @@ public static class JobExecution { Integer resourcesMemoryLimit; OffsetDateTime deployedDate; String deployedBy; - String containerTerminationReason; + String mainContainerTerminationReason; + String initContainerTerminationReason; } @AllArgsConstructor @@ -1456,7 +1459,8 @@ private static String getContainerName(V1Job job) { return (containers != null && !containers.isEmpty()) ? containers.get(0).getName() : null; } - private static Optional getTerminatedState(V1Pod pod) { + private static Optional getTerminatedState( + V1Pod pod, Function> containerStatusFunction) { Objects.requireNonNull(pod, "The pod cannot be null"); var status = pod.getStatus(); if (status == null @@ -1464,7 +1468,7 @@ private static Optional getTerminatedState(V1Pod pod || status.getContainerStatuses().isEmpty()) { return Optional.empty(); } - final var containerStatus = status.getContainerStatuses().get(0); + final var containerStatus = containerStatusFunction.apply(status).get(0); return getTerminatedState(containerStatus.getState()) .or(() -> getTerminatedState(containerStatus.getLastState())); } @@ -1483,7 +1487,8 @@ private static Optional getTerminatedState(V1Contain * @return A {@link V1ContainerStateTerminated} object representing the termination status of the * job. */ - Optional getTerminationStatus(V1Job job) { + ImmutablePair, Optional> + getTerminationStatus(V1Job job) { List jobPods; try { @@ -1493,21 +1498,50 @@ Optional getTerminationStatus(V1Job job) { "Could not list pods for job {}", job.getMetadata().getName(), new KubernetesException("", ex)); - return Optional.empty(); + return ImmutablePair.of(Optional.empty(), Optional.empty()); } - var lastTerminatedPodState = + var lastMainTerminatedPodState = jobPods.stream() - .map(KubernetesService::getTerminatedState) + .map( + v1Pod -> + getTerminatedState( + v1Pod, + new Function>() { + @Override + public List apply(V1PodStatus v1PodStatus) { + return v1PodStatus.getContainerStatuses(); + } + })) .filter(Optional::isPresent) .map(Optional::get) .max(Comparator.comparing(V1ContainerStateTerminated::getFinishedAt)); - if (lastTerminatedPodState.isEmpty()) { - log.info("Could not find a terminated pod for job {}", job.getMetadata().getName()); + if (lastMainTerminatedPodState.isEmpty()) { + log.info("Could not find a main terminated pod for job {}", job.getMetadata().getName()); } - return lastTerminatedPodState; + var lastInitTerminatedPodState = + jobPods.stream() + .map( + v1Pod -> + getTerminatedState( + v1Pod, + new Function>() { + @Override + public List apply(V1PodStatus v1PodStatus) { + return v1PodStatus.getInitContainerStatuses(); + } + })) + .filter(Optional::isPresent) + .map(Optional::get) + .max(Comparator.comparing(V1ContainerStateTerminated::getFinishedAt)); + + if (lastInitTerminatedPodState.isEmpty()) { + log.info("Could not find a data job terminated pod for job {}", job.getMetadata().getName()); + } + + return ImmutablePair.of(lastInitTerminatedPodState, lastMainTerminatedPodState); } /** @@ -1521,23 +1555,35 @@ Optional getJobExecutionStatus(V1Job job, JobStatusCondition jobSt // jobCondition = null means that the K8S Job is still running if (jobStatusCondition != null) { // Job termination status - Optional lastTerminatedPodState = getTerminationStatus(job); + ImmutablePair, Optional> + podTerminationStatus = getTerminationStatus(job); + + Optional lastInitContainerStateTerminated = + podTerminationStatus.getLeft(); + // Termination Reason of the pod init container + lastInitContainerStateTerminated + .map( + v1ContainerStateTerminated -> + StringUtils.trim(v1ContainerStateTerminated.getReason())) + .ifPresent(s -> jobExecutionStatusBuilder.initContainerTerminationReason(s)); + + Optional lastMainContainerStateTerminated = + podTerminationStatus.getRight(); // If the job completed but its pod did not produce a termination message, we infer the - // termination - // status later, based on the status of the job itself. - lastTerminatedPodState + // termination status later, based on the status of the job itself. + lastMainContainerStateTerminated .map( v1ContainerStateTerminated -> StringUtils.trim(v1ContainerStateTerminated.getMessage())) - .ifPresent(s -> jobExecutionStatusBuilder.podTerminationMessage(s)); + .ifPresent(s -> jobExecutionStatusBuilder.mainContainerTerminationMessage(s)); jobExecutionStatusBuilder.jobTerminationReason(jobStatusCondition.getReason()); // Termination Reason of the data job pod container - lastTerminatedPodState + lastMainContainerStateTerminated .map( v1ContainerStateTerminated -> StringUtils.trim(v1ContainerStateTerminated.getReason())) - .ifPresent(s -> jobExecutionStatusBuilder.containerTerminationReason(s)); + .ifPresent(s -> jobExecutionStatusBuilder.mainContainerTerminationReason(s)); } // Job resources @@ -1654,7 +1700,7 @@ Optional getJobExecutionStatus(V1Job job, JobStatusCondition jobSt // omits events that come after the Data Job completion if (jobExecutionStatusBuilder.succeeded != null - && StringUtils.isBlank(jobExecutionStatusBuilder.containerTerminationReason)) { + && StringUtils.isBlank(jobExecutionStatusBuilder.initContainerTerminationReason)) { return Optional.empty(); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java index 4690400e01..ec6c198b6f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionResultManager.java @@ -48,13 +48,13 @@ private static class PodTerminationMessage { */ public static ExecutionResult getResult(KubernetesService.JobExecution jobExecution) { PodTerminationMessage podTerminationMessage = - parsePodTerminationMessage(jobExecution.getPodTerminationMessage()); + parsePodTerminationMessage(jobExecution.getMainContainerTerminationMessage()); ExecutionStatus executionStatus = getExecutionStatus( jobExecution.getSucceeded(), podTerminationMessage.getStatus(), jobExecution.getJobTerminationReason(), - jobExecution.getContainerTerminationReason(), + jobExecution.getMainContainerTerminationReason(), jobExecution.getStartTime()); return ExecutionResult.builder() @@ -78,12 +78,12 @@ public static ExecutionResult getResult(KubernetesService.JobExecution jobExecut * * * @param executionSucceeded K8s Job status (true - succeeded, false - failed, null - running) - * @param podTerminationStatus termination status returned from K8S Pod (e.g. "Success", "User - * error", etc.) + * @param mainContainerTerminationMessage termination status returned from K8S Pod (e.g. + * "Success", "User error", etc.) * @param jobTerminationReason condition reason as reported by K8s Job (e.g. "DeadlineExceeded", * "BackoffLimitExceeded", etc.) - * @param containerTerminationReason termination reason for pod container as returned by K8s pod - * container (e.g., "OOMKilled", etc.) + * @param mainContainerTerminationReason termination reason for pod container as returned by K8s + * pod container (e.g., "OOMKilled", etc.) * @param executionStarTime K8S Job execution start time * @return if there is no termination message due to the missing K8S Pod returns execution status * based on K8S Job status otherwise returns execution status based on the K8S Pod termination @@ -91,9 +91,9 @@ public static ExecutionResult getResult(KubernetesService.JobExecution jobExecut */ private static ExecutionStatus getExecutionStatus( Boolean executionSucceeded, - String podTerminationStatus, + String mainContainerTerminationMessage, String jobTerminationReason, - String containerTerminationReason, + String mainContainerTerminationReason, OffsetDateTime executionStarTime) { ExecutionStatus executionStatus; @@ -101,14 +101,14 @@ private static ExecutionStatus getExecutionStatus( if (executionSucceeded == null) { executionStatus = executionStarTime == null ? ExecutionStatus.SUBMITTED : ExecutionStatus.RUNNING; - } else if (executionSucceeded && StringUtils.isEmpty(podTerminationStatus)) { + } else if (executionSucceeded && StringUtils.isEmpty(mainContainerTerminationMessage)) { executionStatus = ExecutionStatus.SUCCEEDED; - } else if (!executionSucceeded && StringUtils.isEmpty(podTerminationStatus)) { - executionStatus = inferError(jobTerminationReason, containerTerminationReason); + } else if (!executionSucceeded && StringUtils.isEmpty(mainContainerTerminationMessage)) { + executionStatus = inferError(jobTerminationReason, mainContainerTerminationReason); } else { executionStatus = Arrays.stream(ExecutionStatus.values()) - .filter(status -> status.getPodStatus().equals(podTerminationStatus)) + .filter(status -> status.getPodStatus().equals(mainContainerTerminationMessage)) .findAny() .orElse(ExecutionStatus.PLATFORM_ERROR); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java index f3bda17c0e..e8be1d8b70 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java @@ -327,7 +327,7 @@ public Optional updateJobExecu .status(executionStatus) .message( getJobExecutionApiMessage( - executionStatus, jobExecution.getContainerTerminationReason())) + executionStatus, jobExecution.getMainContainerTerminationReason())) .opId(jobExecution.getOpId()) .endTime(jobExecution.getEndTime()) .vdkVersion(executionResult.getVdkVersion()) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DataJobMonitor.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DataJobMonitor.java index 3db824fd32..e768ac44e8 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DataJobMonitor.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DataJobMonitor.java @@ -105,7 +105,7 @@ public void watchJobs() { "Termination message of Data Job {} with execution {}: {}", s.getJobName(), s.getExecutionId(), - s.getPodTerminationMessage()); + s.getMainContainerTerminationMessage()); recordJobExecutionStatus(s); }, runningJobExecutionIds -> { diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java index 918465cb06..e5a2bbeeaa 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java @@ -15,6 +15,7 @@ import com.vmware.taurus.service.model.JobLabel; import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.models.*; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -88,7 +89,8 @@ public void testGetJobExecutionStatus_emptyJob_shouldReturnEmptyJobExecutionStat V1Job v1Job = new V1Job(); var mock = Mockito.mock(KubernetesService.class); Mockito.when(mock.getK8sSupportsV1CronJob()).thenReturn(false); - Mockito.when(mock.getTerminationStatus(v1Job)).thenReturn(Optional.empty()); + Mockito.when(mock.getTerminationStatus(v1Job)) + .thenReturn(ImmutablePair.of(Optional.empty(), Optional.empty())); Mockito.when(mock.getJobExecutionStatus(v1Job, null)).thenCallRealMethod(); Optional actualJobExecutionStatusOptional = mock.getJobExecutionStatus(v1Job, null); @@ -175,7 +177,10 @@ public void testGetJobExecutionStatus_notEmptyJob_shouldReturnNotEmptyJobExecuti KubernetesService mock = Mockito.mock(KubernetesService.class); Mockito.when(mock.getK8sSupportsV1CronJob()).thenReturn(false); Mockito.when(mock.getTerminationStatus(expectedJob)) - .thenReturn(Optional.ofNullable(new V1ContainerStateTerminated().reason("test"))); + .thenReturn( + ImmutablePair.of( + Optional.ofNullable(new V1ContainerStateTerminated().reason("test")), + Optional.ofNullable(new V1ContainerStateTerminated().reason("test")))); Mockito.when(mock.getJobExecutionStatus(expectedJob, condition)).thenCallRealMethod(); Optional actualJobExecutionStatusOptional = mock.getJobExecutionStatus(expectedJob, condition); diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionResultManagerTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionResultManagerTest.java index 662c135c16..d5f875b90d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionResultManagerTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionResultManagerTest.java @@ -15,8 +15,6 @@ import com.vmware.taurus.service.model.ExecutionStatus; import com.vmware.taurus.service.model.ExecutionResult; -import java.time.OffsetDateTime; - public class JobExecutionResultManagerTest { @Test @@ -24,7 +22,7 @@ public void testGetResult_succeededTrueAndNullTerminationMessage_shouldReturnMes KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage(null) + .mainContainerTerminationMessage(null) .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -38,7 +36,7 @@ public void testGetResult_succeededFalseAndNullTerminationMessage_shouldReturnMe KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage(null) + .mainContainerTerminationMessage(null) .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -54,7 +52,7 @@ public void testGetResult_succeededFalseAndNullTerminationMessage_shouldReturnMe KubernetesService.JobExecution.builder() .succeeded(null) .startTime(OffsetDateTime.now()) - .podTerminationMessage(null) + .mainContainerTerminationMessage(null) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -64,7 +62,10 @@ public void testGetResult_succeededFalseAndNullTerminationMessage_shouldReturnMe @Test public void testGetResult_succeededTrueAndEmptyTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = - KubernetesService.JobExecution.builder().succeeded(true).podTerminationMessage("").build(); + KubernetesService.JobExecution.builder() + .succeeded(true) + .mainContainerTerminationMessage("") + .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); Assertions.assertEquals( @@ -75,7 +76,10 @@ public void testGetResult_succeededTrueAndEmptyTerminationMessage_shouldReturnMe @Test public void testGetResult_succeededFalseAndEmptyTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = - KubernetesService.JobExecution.builder().succeeded(false).podTerminationMessage("").build(); + KubernetesService.JobExecution.builder() + .succeeded(false) + .mainContainerTerminationMessage("") + .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); Assertions.assertEquals( @@ -90,7 +94,7 @@ public void testGetResult_succeededFalseAndEmptyTerminationMessage_shouldReturnM KubernetesService.JobExecution.builder() .succeeded(null) .startTime(OffsetDateTime.now()) - .podTerminationMessage("") + .mainContainerTerminationMessage("") .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -102,7 +106,7 @@ public void testGetResult_succeededTrueAndNotExistingTerminationMessage_shouldRe KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage("not-existing-termination-message") + .mainContainerTerminationMessage("not-existing-termination-message") .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -116,7 +120,7 @@ public void testGetResult_succeededFalseAndNotExistingTerminationMessage_shouldR KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage("not-existing-termination-message") + .mainContainerTerminationMessage("not-existing-termination-message") .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -132,7 +136,7 @@ public void testGetResult_succeededNullAndNotExistingTerminationMessage_shouldRe KubernetesService.JobExecution.builder() .succeeded(null) .startTime(OffsetDateTime.now()) - .podTerminationMessage("not-existing-termination-message") + .mainContainerTerminationMessage("not-existing-termination-message") .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -154,7 +158,7 @@ public void testGetResult_jsonTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage(expectedTerminationMessage.toString()) + .mainContainerTerminationMessage(expectedTerminationMessage.toString()) .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -169,7 +173,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage(expectedTerminationStatus.getPodStatus()) + .mainContainerTerminationMessage(expectedTerminationStatus.getPodStatus()) .build(); ExecutionResult actualTerminationMessage = JobExecutionResultManager.getResult(jobExecution); @@ -184,7 +188,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage(expectedExecutionStatus.getPodStatus()) + .mainContainerTerminationMessage(expectedExecutionStatus.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -198,7 +202,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage(expectedExecutionStatus.getPodStatus()) + .mainContainerTerminationMessage(expectedExecutionStatus.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -212,7 +216,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution.builder() .succeeded(null) .startTime(OffsetDateTime.now()) - .podTerminationMessage(ExecutionStatus.USER_ERROR.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.USER_ERROR.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -226,7 +230,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage(expectedExecutionStatus.getPodStatus()) + .mainContainerTerminationMessage(expectedExecutionStatus.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -240,7 +244,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage(expectedExecutionStatus.getPodStatus()) + .mainContainerTerminationMessage(expectedExecutionStatus.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -254,7 +258,7 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { KubernetesService.JobExecution.builder() .succeeded(null) .startTime(OffsetDateTime.now()) - .podTerminationMessage(ExecutionStatus.PLATFORM_ERROR.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.PLATFORM_ERROR.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -265,7 +269,10 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { public void testGetResult_emptyTerminationMessageAndExecutionStatusSucceeded_shouldReturnTerminationStatusSuccess() { KubernetesService.JobExecution jobExecution = - KubernetesService.JobExecution.builder().succeeded(true).podTerminationMessage("").build(); + KubernetesService.JobExecution.builder() + .succeeded(true) + .mainContainerTerminationMessage("") + .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); Assertions.assertEquals(ExecutionStatus.SUCCEEDED, actualResult.getExecutionStatus()); @@ -275,7 +282,10 @@ public void testGetResult_stringTerminationMessage_shouldReturnMessage() { public void testGetResult_emptyTerminationMessageAndExecutionStatusFailed_shouldReturnTerminationStatusPlatformError() { KubernetesService.JobExecution jobExecution = - KubernetesService.JobExecution.builder().succeeded(false).podTerminationMessage("").build(); + KubernetesService.JobExecution.builder() + .succeeded(false) + .mainContainerTerminationMessage("") + .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); Assertions.assertEquals(ExecutionStatus.PLATFORM_ERROR, actualResult.getExecutionStatus()); @@ -328,7 +338,7 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage(ExecutionStatus.USER_ERROR.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.USER_ERROR.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -341,7 +351,7 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage(ExecutionStatus.SKIPPED.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.SKIPPED.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -354,7 +364,7 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(true) - .podTerminationMessage(ExecutionStatus.PLATFORM_ERROR.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.PLATFORM_ERROR.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -367,7 +377,7 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() .succeeded(false) - .podTerminationMessage(ExecutionStatus.PLATFORM_ERROR.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.PLATFORM_ERROR.getPodStatus()) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); @@ -379,7 +389,7 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai testGetResult_terminationMessageNullAndExecutionStatusFailedAndTerminationReasonDeadlineExceeded_shouldReturnTerminationStatusUserError() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() - .podTerminationMessage(null) + .mainContainerTerminationMessage(null) .succeeded(false) .jobTerminationReason(JobExecutionResultManager.TERMINATION_REASON_DEADLINE_EXCEEDED) .build(); @@ -393,7 +403,7 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai testGetResult_terminationMessageNullAndExecutionStatusFailedAndTerminationReasonAny_shouldReturnTerminationStatusUserError() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() - .podTerminationMessage(null) + .mainContainerTerminationMessage(null) .succeeded(false) .jobTerminationReason("SomeReason") .build(); @@ -407,10 +417,11 @@ public void testGetResult_executionSucceededFalse_shouldReturnExecutionStatusFai testGetResult_terminationMessageNullAndExecutionStatusFailedAndTerminationReasonOutOfMemory_shouldReturnTerminationStatusUserError() { KubernetesService.JobExecution jobExecution = KubernetesService.JobExecution.builder() - .podTerminationMessage(null) + .mainContainerTerminationMessage(null) .succeeded(false) .jobTerminationReason("Some Reason") - .containerTerminationReason(JobExecutionResultManager.TERMINATION_REASON_OUT_OF_MEMORY) + .mainContainerTerminationReason( + JobExecutionResultManager.TERMINATION_REASON_OUT_OF_MEMORY) .build(); ExecutionResult actualResult = JobExecutionResultManager.getResult(jobExecution); diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionServiceUpdateExecutionIT.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionServiceUpdateExecutionIT.java index 4751563213..cd590af44a 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionServiceUpdateExecutionIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/execution/JobExecutionServiceUpdateExecutionIT.java @@ -236,8 +236,8 @@ private KubernetesService.JobExecution createJobExecution( .resourcesMemoryRequest(1) .deployedBy("test_deployed_by") .deployedDate(getTimeAccurateToMicroSecond()) - .podTerminationMessage(terminationMessage) - .containerTerminationReason(containerTerminationMessage) + .mainContainerTerminationMessage(terminationMessage) + .mainContainerTerminationReason(containerTerminationMessage) .build(); ExecutionResult executionResult = JobExecutionResultManager.getResult(jobExecution); jobExecutionService.updateJobExecution(dataJob, jobExecution, executionResult); @@ -272,7 +272,7 @@ private void testUpdateJobExecution( expectedJobExecutionStatus, expectedJobExecutionMessage != null ? expectedJobExecutionMessage - : expectedJobExecution.getPodTerminationMessage(), + : expectedJobExecution.getMainContainerTerminationMessage(), actualJobExecution, expectedVdkVersion); } @@ -398,7 +398,7 @@ private void testUpdateJobExecutionWithPreviousStatusInDatabase( .resourcesMemoryRequest(1) .deployedBy("test_deployed_by") .deployedDate(OffsetDateTime.now()) - .podTerminationMessage(expectedStatus.getPodStatus()) + .mainContainerTerminationMessage(expectedStatus.getPodStatus()) .build(); ExecutionResult executionResult = JobExecutionResultManager.getResult(attemptedExecutionUpdate); jobExecutionService.updateJobExecution( diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/monitoring/DataJobMonitorTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/monitoring/DataJobMonitorTest.java index fa0a498667..11d6ad846d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/monitoring/DataJobMonitorTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/monitoring/DataJobMonitorTest.java @@ -406,7 +406,7 @@ public void testRecordJobExecutionStatus_emptyDataJobName_shouldNotRecordExecuti JobExecution.builder() .jobName(jobExecutionBeforeUpdate.getDataJob().getName()) .executionId(jobExecutionBeforeUpdate.getId()) - .podTerminationMessage(ExecutionStatus.SUCCEEDED.getPodStatus()) + .mainContainerTerminationMessage(ExecutionStatus.SUCCEEDED.getPodStatus()) .executionType("scheduled") .opId("opId") .startTime(jobExecutionBeforeUpdate.getStartTime()) @@ -858,7 +858,7 @@ private static JobExecution buildJobExecutionStatus( return JobExecution.builder() .jobName(jobName) .executionId(executionId) - .podTerminationMessage(terminationMessage) + .mainContainerTerminationMessage(terminationMessage) .executionType("scheduled") .opId("opId") .startTime(startTime) @@ -923,7 +923,7 @@ private void assertDataJobExecutionValid( actualDataJobExecution.getResourcesMemoryLimit()); Assertions.assertEquals( expectedExecutionMessage == null - ? expectedJobExecution.getPodTerminationMessage() + ? expectedJobExecution.getMainContainerTerminationMessage() : expectedExecutionMessage, actualDataJobExecution.getMessage()); Assertions.assertEquals(