vmware · murphp15 · Jan 25, 2023 · Dec 13, 2022 · Dec 14, 2022 · Dec 14, 2022
diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml
@@ -33,7 +33,7 @@
   variables:
     _JAVA_OPTIONS: "-Xms2048m -Xmx4096m"
   before_script:
-    - apk --no-cache add openjdk11-jdk git zip curl zip py-pip
+    - apk --no-cache add openjdk17-jdk git zip curl zip py-pip
     - pip install --upgrade pip && pip install awscli
 
 control_service_build_image:
@@ -42,7 +42,7 @@ control_service_build_image:
     - .images:dind
   stage: build
   script:
-    - apk --no-cache add git openjdk11-jdk
+    - apk --no-cache add git openjdk17-jdk
     - export TAG=$(git rev-parse --short HEAD)
     - cd projects/control-service/projects
     - ./gradlew -p ./model build publishToMavenLocal --info --stacktrace
@@ -197,7 +197,7 @@ control_service_publish_image:
   extends: .images:dind
   stage: publish_artifacts
   script:
-    - apk add --no-cache git openjdk11-jdk
+    - apk add --no-cache git openjdk17-jdk
     - export TAG=$(git rev-parse --short HEAD)
     - docker login -u $VDK_DOCKER_REGISTRY_USERNAME -p $VDK_DOCKER_REGISTRY_PASSWORD
     - cd projects/control-service/projects
@@ -222,7 +222,7 @@ control_service_publish_api_client:
   image: docker:20.10.22
   stage: publish_artifacts
   script:
-    - apk add --no-cache py-pip openjdk11-jdk git python
+    - apk add --no-cache py-pip openjdk17-jdk git python
     - pip install -U pip setuptools wheel twine
     - ./projects/control-service/projects/gradlew -p ./projects/control-service/projects/model buildPython
     - cd projects/control-service/projects/model/apidefs/datajob-api/build
@@ -252,14 +252,10 @@ control_service_deploy_testing_data_pipelines:
     - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
     - bash -ex ./projects/control-service/cicd/deploy-testing-pipelines-service.sh
   retry: !reference [.control_service_retry, retry_options]
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes: [ projects/control-service/projects/helm_charts/pipelines-control-service/version.txt ]
-      when: never
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes: *control_service_code_change_locations
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes: *control_service_helm_change_locations
+  only:
+    refs:
+      #  TODO revert before merging
+      - external_pull_requests
 
 # vdk-heartbeat source: https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-heartbeat
 control_service_post_deployment_test:
@@ -274,14 +270,10 @@ control_service_post_deployment_test:
     - export VDK_HEARTBEAT_OP_ID="vdkcs-$CI_PIPELINE_ID"
     - vdk-heartbeat -f projects/control-service/cicd/post_deploy_test_heartbeat_config.ini
   retry: !reference [.control_service_retry, retry_options]
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes: [ projects/control-service/projects/helm_charts/pipelines-control-service/version.txt ]
-      when: never
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes: *control_service_code_change_locations
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes: *control_service_helm_change_locations
+  only:
+    refs:
+      #  TODO revert before merging
+      - external_pull_requests
 
 control_service_release:
   image: docker:20.10.22

diff --git a/projects/control-service/projects/java-common.gradle b/projects/control-service/projects/java-common.gradle
@@ -1,5 +1,5 @@
 apply plugin: 'java'
 apply plugin: 'idea'
-sourceCompatibility = '11'
-targetCompatibility = '11'
+sourceCompatibility = '17'
+targetCompatibility = '17'
 group 'com.vmware.taurus'
diff --git a/projects/control-service/projects/pipelines_control_service/Dockerfile b/projects/control-service/projects/pipelines_control_service/Dockerfile
@@ -4,15 +4,15 @@
 # Builder stage
 # This stage is implemented like in the guide, but it may be more efficient to
 # have it in the gradle build process outside docker.
-FROM adoptopenjdk:11-jre-hotspot as builder
+FROM eclipse-temurin:17 as builder
 ARG JAR_NAME
 WORKDIR application
 COPY ./${JAR_NAME} application.jar
 RUN java -Djarmode=layertools -jar application.jar extract
 
 
 # Final stage
-FROM adoptopenjdk:11-jre-hotspot
+FROM eclipse-temurin:17
 LABEL maintainer="Project Versatile Data Kit <[email protected]>"
 
 # install kerberos client so that we can use kadmin

diff --git a/...vice/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java b/...vice/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java
@@ -6,6 +6,7 @@
 package com.vmware.taurus.datajobs.it.common;
 
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 
 import com.vmware.taurus.service.JobExecutionRepository;
 import com.vmware.taurus.service.model.DataJob;
@@ -15,6 +16,16 @@
 
 public class JobExecutionUtil {
 
+  /**
+   * at the database level we only store date-times accurate to the microsecond. Like wise in older
+   * versions of java .now() returned timestamps accurate to micro-seconds. In newer versions of
+   * java .now() gives nano-second precision and it causes tests written before we adopted that java
+   * version to fail.
+   */
+  public static OffsetDateTime getTimeAccurateToMicroSecond() {
+    return OffsetDateTime.now().truncatedTo(ChronoUnit.MICROS);
+  }
+
   public static DataJobExecution createDataJobExecution(
       JobExecutionRepository jobExecutionRepository,
       String executionId,
@@ -37,7 +48,7 @@ public static DataJobExecution createDataJobExecution(
             .resourcesMemoryLimit(1000)
             .message("message")
             .lastDeployedBy("test_user")
-            .lastDeployedDate(OffsetDateTime.now())
+            .lastDeployedDate(getTimeAccurateToMicroSecond())
             .jobVersion("test_version")
             .jobSchedule("*/5 * * * *")
             .opId("test_op_id")

diff --git a/...l_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java b/...l_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java
@@ -24,6 +24,7 @@
 import java.time.OffsetDateTime;
 import java.util.List;
 
+import static com.vmware.taurus.datajobs.it.common.JobExecutionUtil.getTimeAccurateToMicroSecond;
 import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
@@ -67,7 +68,7 @@ public void setup() {
     this.dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, config2));
     this.dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, config3));
 
-    OffsetDateTime now = OffsetDateTime.now();
+    OffsetDateTime now = getTimeAccurateToMicroSecond();
     this.dataJobExecution1 =
         JobExecutionUtil.createDataJobExecution(
             jobExecutionRepository, "testId1", dataJob1, now, now, ExecutionStatus.SUCCEEDED);
@@ -101,9 +102,8 @@ private static String getQuery() {
   }
 
   private void cleanup() {
-    jobsRepository
-        .findAllById(List.of(TEST_JOB_NAME_1, TEST_JOB_NAME_2, TEST_JOB_NAME_3))
-        .forEach(dataJob -> jobsRepository.delete(dataJob));
+    jobsRepository.deleteAll(
+        jobsRepository.findAllById(List.of(TEST_JOB_NAME_1, TEST_JOB_NAME_2, TEST_JOB_NAME_3)));
   }
 
   @Test

diff --git a/...ce/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java b/...ce/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java
@@ -12,6 +12,7 @@
 
 import java.text.MessageFormat;
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 
 import org.hamcrest.Matchers;
@@ -132,7 +133,7 @@ public void testExecutions_filterByStartTimeGte() throws Exception {
                     "{"
                         + "\"filter\": {"
                         + "      \"startTimeGte\": \""
-                        + dataJobExecution3.getStartTime()
+                        + dataJobExecution3.getStartTime().truncatedTo(ChronoUnit.MICROS)
                         + "\""
                         + "    },"
                         + "\"pageNumber\": 1,"

diff --git a/...tegration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java b/...tegration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java
@@ -11,6 +11,7 @@
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.UUID;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -95,7 +96,7 @@ public void testCallWithSingleExecution(String jobName, String username) throws
         .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId))
         .andExpect(
             jsonPath("$.data.content[0].deployments[0].executions[0].endTime")
-                .value(expectedEndTime.toString()));
+                .value(roundDateTimeToMicros(expectedEndTime)));
   }
 
   @Test
@@ -120,11 +121,11 @@ public void testCallTwoExecutionsSortAsc(String jobName, String username) throws
         .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId2))
         .andExpect(
             jsonPath("$.data.content[0].deployments[0].executions[0].endTime")
-                .value(expectedEndTimeSmaller.toString()))
+                .value(roundDateTimeToMicros(expectedEndTimeSmaller)))
         .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1].id").value(expectedId1))
         .andExpect(
             jsonPath("$.data.content[0].deployments[0].executions[1].endTime")
-                .value(expectedEndTimeLarger.toString()));
+                .value(roundDateTimeToMicros(expectedEndTimeLarger)));
   }
 
   @Test
@@ -149,11 +150,11 @@ public void testCallTwoExecutionsSortDesc(String jobName, String username) throw
         .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId1))
         .andExpect(
             jsonPath("$.data.content[0].deployments[0].executions[0].endTime")
-                .value(expectedEndTimeLarger.toString()))
+                .value(roundDateTimeToMicros(expectedEndTimeLarger)))
         .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1].id").value(expectedId2))
         .andExpect(
             jsonPath("$.data.content[0].deployments[0].executions[1].endTime")
-                .value(expectedEndTimeSmaller.toString()));
+                .value(roundDateTimeToMicros(expectedEndTimeSmaller)));
   }
 
   @Test
@@ -200,4 +201,8 @@ private void createDataJobExecution(String executionId, String jobName, OffsetDa
 
     jobExecutionRepository.save(jobExecution);
   }
+
+  private static String roundDateTimeToMicros(OffsetDateTime dateTime) {
+    return dateTime.plusNanos(500).truncatedTo(ChronoUnit.MICROS).toString();
+  }
 }
diff --git a/...ce/projects/pipelines_control_service/src/test/java/com/vmware/taurus/RepositoryUtil.java b/...ce/projects/pipelines_control_service/src/test/java/com/vmware/taurus/RepositoryUtil.java
@@ -16,6 +16,7 @@
 import org.junit.jupiter.api.Assertions;
 
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 
 public final class RepositoryUtil {
 
@@ -82,7 +83,7 @@ public static DataJobExecution createDataJobExecution(
         executionStatus,
         message,
         startTime,
-        OffsetDateTime.now());
+        getTimeAccurateToMicroSecond());
   }
 
   public static DataJobExecution createDataJobExecution(
@@ -99,7 +100,7 @@ public static DataJobExecution createDataJobExecution(
         executionStatus,
         "test_message",
         startTime,
-        OffsetDateTime.now());
+        getTimeAccurateToMicroSecond());
   }
 
   public static DataJobExecution createDataJobExecution(
@@ -115,8 +116,8 @@ public static DataJobExecution createDataJobExecution(
         dataJob,
         executionStatus,
         message,
-        OffsetDateTime.now(),
-        OffsetDateTime.now());
+        getTimeAccurateToMicroSecond(),
+        getTimeAccurateToMicroSecond());
   }
 
   public static DataJobExecution createDataJobExecution(
@@ -142,7 +143,7 @@ public static DataJobExecution createDataJobExecution(
             .resourcesMemoryLimit(1000)
             .message(message)
             .lastDeployedBy("test_user")
-            .lastDeployedDate(OffsetDateTime.now())
+            .lastDeployedDate(getTimeAccurateToMicroSecond())
             .jobVersion("test_version")
             .jobSchedule("*/5 * * * *")
             .opId("test_op_id")
@@ -151,4 +152,14 @@ public static DataJobExecution createDataJobExecution(
 
     return jobExecutionRepository.save(expectedJobExecution);
   }
+
+  /**
+   * at the database level we only store date-times accurate to the microsecond. Like wise in older
+   * versions of java .now() returned timestamps accurate to micro-seconds. In newer versions of
+   * java .now() gives nano-second precision and it causes tests written before we adopted that java
+   * version to fail.
+   */
+  public static OffsetDateTime getTimeAccurateToMicroSecond() {
+    return OffsetDateTime.now().truncatedTo(ChronoUnit.MICROS);
+  }
 }
diff --git a/...nes_control_service/src/test/java/com/vmware/taurus/service/JobExecutionFilterSpecIT.java b/...nes_control_service/src/test/java/com/vmware/taurus/service/JobExecutionFilterSpecIT.java
@@ -20,6 +20,8 @@
 import java.time.OffsetDateTime;
 import java.util.List;
 
+import static com.vmware.taurus.RepositoryUtil.getTimeAccurateToMicroSecond;
+
 @SpringBootTest(classes = ControlplaneApplication.class)
 public class JobExecutionFilterSpecIT {
 
@@ -69,7 +71,7 @@ public void testJobExecutionFilterSpec_filerByStatusIn_shouldReturnResult() {
   @Test
   public void testJobExecutionFilterSpec_filerByStartTimeGte_shouldReturnResult() {
     DataJob actualDataJob = RepositoryUtil.createDataJob(jobsRepository);
-    OffsetDateTime now = OffsetDateTime.now();
+    OffsetDateTime now = getTimeAccurateToMicroSecond();
 
     RepositoryUtil.createDataJobExecution(
         jobExecutionRepository,
@@ -112,7 +114,7 @@ public void testJobExecutionFilterSpec_filerByStartTimeGte_shouldReturnResult()
   @Test
   public void testJobExecutionFilterSpec_filerByEndTimeGte_shouldReturnResult() {
     DataJob actualDataJob = RepositoryUtil.createDataJob(jobsRepository);
-    OffsetDateTime now = OffsetDateTime.now();
+    OffsetDateTime now = getTimeAccurateToMicroSecond();
 
     RepositoryUtil.createDataJobExecution(
         jobExecutionRepository,
@@ -159,7 +161,7 @@ public void testJobExecutionFilterSpec_filerByEndTimeGte_shouldReturnResult() {
   @Test
   public void testJobExecutionFilterSpec_filerByAllFields_shouldReturnResult() {
     DataJob actualDataJob = RepositoryUtil.createDataJob(jobsRepository);
-    OffsetDateTime now = OffsetDateTime.now();
+    OffsetDateTime now = getTimeAccurateToMicroSecond();
 
     RepositoryUtil.createDataJobExecution(
         jobExecutionRepository,

diff --git a/...nes_control_service/src/test/java/com/vmware/taurus/service/JobExecutionRepositoryIT.java b/...nes_control_service/src/test/java/com/vmware/taurus/service/JobExecutionRepositoryIT.java
@@ -23,6 +23,8 @@
 import java.time.ZoneOffset;
 import java.util.List;
 
+import static com.vmware.taurus.RepositoryUtil.getTimeAccurateToMicroSecond;
+
 /** Integration tests of the setup of Spring Data repository for data job executions */
 @SpringBootTest(classes = ControlplaneApplication.class)
 public class JobExecutionRepositoryIT {
@@ -246,7 +248,7 @@ void testUpdateExecutionStatusWhereOldStatusInAndExecutionIdIn_withExecutions_sh
     jobExecutionRepository.save(execution1);
     jobExecutionRepository.save(execution2);
 
-    var executionEndTime = OffsetDateTime.now();
+    var executionEndTime = getTimeAccurateToMicroSecond();
     var message = "Changed by test";
     jobExecutionRepository.updateExecutionStatusWhereOldStatusInAndExecutionIdIn(
         ExecutionStatus.CANCELLED,
@@ -293,7 +295,7 @@ void testUpdateExecutionStatusWhereOldStatusInAndExecutionIdIn_withExecutions_sh
     jobExecutionRepository.save(execution1);
     jobExecutionRepository.save(execution2);
 
-    var executionEndTime = OffsetDateTime.now();
+    var executionEndTime = getTimeAccurateToMicroSecond();
     var message = "Changed by test";
     jobExecutionRepository.updateExecutionStatusWhereOldStatusInAndExecutionIdIn(
         ExecutionStatus.CANCELLED,

diff --git a/...c/test/java/com/vmware/taurus/service/execution/JobExecutionServiceUpdateExecutionIT.java b/...c/test/java/com/vmware/taurus/service/execution/JobExecutionServiceUpdateExecutionIT.java
@@ -29,6 +29,8 @@
 import com.vmware.taurus.service.model.ExecutionStatus;
 import com.vmware.taurus.service.model.ExecutionResult;
 
+import static com.vmware.taurus.RepositoryUtil.getTimeAccurateToMicroSecond;
+
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = ControlplaneApplication.class)
 public class JobExecutionServiceUpdateExecutionIT {
@@ -233,7 +235,7 @@ private KubernetesService.JobExecution createJobExecution(
             .resourcesMemoryLimit(1)
             .resourcesMemoryRequest(1)
             .deployedBy("test_deployed_by")
-            .deployedDate(OffsetDateTime.now())
+            .deployedDate(getTimeAccurateToMicroSecond())
             .podTerminationMessage(terminationMessage)
             .containerTerminationReason(containerTerminationMessage)
             .build();
@@ -256,8 +258,8 @@ private void testUpdateJobExecution(
             actualDataJob,
             actualExecutionSucceeded,
             actualTerminationMessage,
-            OffsetDateTime.now(),
-            OffsetDateTime.now(),
+            getTimeAccurateToMicroSecond(),
+            getTimeAccurateToMicroSecond(),
             "");
     DataJobExecution actualJobExecution =
         jobExecutionService.readJobExecution(