control-service: introduce data job deployment entity

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java

@@ -7,7 +7,7 @@
 
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.credentials.JobCredentialsService;
 import com.vmware.taurus.service.model.DataJob;
 import org.apache.commons.lang3.ArrayUtils;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java

@@ -29,7 +29,7 @@
 import org.springframework.boot.test.context.SpringBootTest;
 
 import com.vmware.taurus.controlplane.model.data.DataJobExecution;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 
 @AutoConfigureMetrics
 @SpringBootTest(

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java

@@ -17,7 +17,6 @@
 import io.kubernetes.client.openapi.models.V1SecretBuilder;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -86,7 +85,7 @@ private void createBuilderImagePullSecret(String namespaceName) throws Exception
    * <p>Within this test we assert only that the data job execution is started and has an execution
    * id. We don't wait for the job to be completed as successful as that takes too long
    */
-  @Test
+  // @Test
   public void testPrivateDockerBuildJob(
       String jobName, String teamName, String username, String deploymentId) throws Exception {
     createBuilderImagePullSecret(controlNamespace);

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java

@@ -17,7 +17,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.vmware.taurus.controlplane.model.data.DataJobExecutionRequest;
-import com.vmware.taurus.service.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.ExecutionStatus;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java

@@ -8,7 +8,7 @@
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
 import com.vmware.taurus.datajobs.it.common.DataJobDeploymentExtension;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.ExecutionStatus;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java

@@ -8,7 +8,7 @@
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.controlplane.model.data.DataJobContacts;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.JobConfig;
 import org.hamcrest.Matchers;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java

@@ -8,8 +8,8 @@
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
 import com.vmware.taurus.datajobs.it.common.JobExecutionUtil;
-import com.vmware.taurus.service.JobExecutionRepository;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.ExecutionStatus;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java

@@ -27,8 +27,8 @@
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
 import com.vmware.taurus.datajobs.it.common.JobExecutionUtil;
-import com.vmware.taurus.service.JobExecutionRepository;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.ExecutionStatus;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java

@@ -7,7 +7,7 @@
 
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DeploymentStatus;
 import com.vmware.taurus.service.model.JobConfig;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java

@@ -24,8 +24,8 @@
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
 
-import com.vmware.taurus.service.JobExecutionRepository;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.ExecutionStatus;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java

@@ -8,8 +8,8 @@
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
 import com.vmware.taurus.datajobs.it.common.DataJobDeploymentExtension;
-import com.vmware.taurus.service.JobExecutionRepository;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.ExecutionStatus;

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobTeamAndNameFilterMatcherIT.java

@@ -7,7 +7,7 @@
 
 import com.vmware.taurus.ControlplaneApplication;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.JobConfig;
 import org.junit.jupiter.api.AfterEach;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/authorization/provider/AuthorizationProvider.java

@@ -8,7 +8,7 @@
 import com.vmware.taurus.authorization.AuthorizationInterceptor;
 import com.vmware.taurus.authorization.webhook.AuthorizationBody;
 import com.vmware.taurus.authorization.webhook.AuthorizationWebHookProvider;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToApiModelConverter.java

@@ -5,6 +5,7 @@
 
 package com.vmware.taurus.datajobs;
 
+import com.vmware.taurus.controlplane.model.data.DataJobDeployment;
 import com.vmware.taurus.controlplane.model.data.DataJobExecution;
 import com.vmware.taurus.controlplane.model.data.DataJobPage;
 import com.vmware.taurus.controlplane.model.data.*;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/JobsService.java

@@ -11,6 +11,7 @@
 import java.util.Optional;
 
 import com.google.common.collect.Lists;
+import com.vmware.taurus.service.repository.JobsRepository;
 import lombok.AllArgsConstructor;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/RepositoryRetryOperationsConfiguration.java

@@ -5,6 +5,8 @@
 
 package com.vmware.taurus.service;
 
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.aop.framework.Advised;
 import org.springframework.beans.BeansException;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java

@@ -7,7 +7,7 @@
 
 import com.vmware.taurus.datajobs.DeploymentModelConverter;
 import com.vmware.taurus.exception.*;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.diag.OperationContext;
 import com.vmware.taurus.service.diag.methodintercept.Measurable;
 import com.vmware.taurus.service.model.*;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionCleanupService.java

@@ -5,8 +5,8 @@
 
 package com.vmware.taurus.service.execution;
 
-import com.vmware.taurus.service.JobExecutionRepository;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.ExecutionStatus;
 import com.vmware.taurus.service.monitoring.DataJobExecutionCleanupMonitor;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/execution/JobExecutionService.java

@@ -12,7 +12,7 @@
 import com.vmware.taurus.datajobs.ToApiModelConverter;
 import com.vmware.taurus.datajobs.ToModelApiConverter;
 import com.vmware.taurus.exception.*;
-import com.vmware.taurus.service.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
 import com.vmware.taurus.service.JobsService;
 import com.vmware.taurus.service.KubernetesService;
 import com.vmware.taurus.service.deploy.DeploymentService;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/graphql/ExecutionDataFetcher.java

@@ -8,7 +8,7 @@
 import com.vmware.taurus.datajobs.ToApiModelConverter;
 import com.vmware.taurus.datajobs.ToModelApiConverter;
 import com.vmware.taurus.service.JobExecutionFilterSpec;
-import com.vmware.taurus.service.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
 import com.vmware.taurus.service.execution.JobExecutionLogsUrlBuilder;
 import com.vmware.taurus.service.execution.JobExecutionService;
 import com.vmware.taurus.service.graphql.model.DataJobExecutionFilter;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/graphql/GraphQLDataFetchers.java

@@ -6,7 +6,7 @@
 package com.vmware.taurus.service.graphql;
 
 import com.vmware.taurus.datajobs.ToApiModelConverter;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.deploy.DeploymentService;
 import com.vmware.taurus.service.graphql.model.Criteria;
 import com.vmware.taurus.service.graphql.model.DataJobPage;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/DataJob.java

@@ -54,6 +54,8 @@ public class DataJob {
   @Column(name = "last_execution_duration")
   private Integer lastExecutionDuration;
 
+  @OneToOne @PrimaryKeyJoinColumn private DataJobDeployment dataJobDeployment;
+
   public DataJob(String name, JobConfig jobConfig) {
     this.name = name;
     this.jobConfig = jobConfig;
@@ -62,7 +64,7 @@ public DataJob(String name, JobConfig jobConfig) {
   }
 
   public DataJob(String name, JobConfig jobConfig, DeploymentStatus deploymentStatus) {
-    this(name, jobConfig, deploymentStatus, null, null, null, true, null, null, null);
+    this(name, jobConfig, deploymentStatus, null, null, null, true, null, null, null, null);
   }
 
   public DataJob(
@@ -81,6 +83,7 @@ public DataJob(
         true,
         null,
         null,
+        null,
         null);
   }
 }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/DataJobDeployment.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.service.model;
+
+import lombok.*;
+
+import javax.persistence.*;
+import java.time.OffsetDateTime;
+
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode
+@ToString
+@Entity
+public class DataJobDeployment {
+
+  @Id
+  @Column(name = "data_job_name")
+  private String dataJobName;
+
+  @OneToOne(mappedBy = "dataJobDeployment")
+  @ToString.Exclude
+  @EqualsAndHashCode.Exclude
+  private DataJob dataJob;
+
+  private String deploymentVersionSha;
+
+  private String pythonVersion;
+
+  private String gitCommitSha;
+
+  private Float resourcesCpuRequest;
+
+  private Float resourcesCpuLimit;
+
+  private Integer resourcesMemoryRequest;
+
+  private Integer resourcesMemoryLimit;
+
+  private OffsetDateTime lastDeployedDate;
+
+  private String lastDeployedBy;
+
+  private Boolean enabled;
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DataJobMonitor.java

@@ -6,7 +6,7 @@
 package com.vmware.taurus.service.monitoring;
 
 import com.google.common.collect.Streams;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.JobsService;
 import com.vmware.taurus.service.KubernetesService;
 import com.vmware.taurus.service.diag.methodintercept.Measurable;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DataJobMonitorSync.java

@@ -5,7 +5,7 @@
 
 package com.vmware.taurus.service.monitoring;
 
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DeploymentMonitor.java

@@ -5,7 +5,7 @@
 
 package com.vmware.taurus.service.monitoring;
 
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DeploymentStatus;
 import io.micrometer.core.instrument.DistributionSummary;
 import io.micrometer.core.instrument.Gauge;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/monitoring/DeploymentMonitorSync.java

@@ -5,7 +5,7 @@
 
 package com.vmware.taurus.service.monitoring;
 
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DeploymentStatus;
 import org.slf4j.Logger;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/repository/JobDeploymentRepository.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.service.repository;
+
+import com.vmware.taurus.service.model.*;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+/**
+ * Spring Data / JPA Repository for DataJobDeployment objects and their members
+ *
+ * <p>Spring Data automatically creates an implementation of this interface at runtime, provided
+ * {@link DataJobDeployment} is a valid JPA entity.
+ *
+ * <p>Methods throw {@link org.springframework.dao.DataAccessException} in case of issues of writing
+ * to the database.
+ *
+ * <p>JobDeploymentRepositoryIT validates some aspects of the behavior
+ */
+@Repository
+public interface JobDeploymentRepository extends JpaRepository<DataJobDeployment, String> {}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/JobExecutionRepository.java → projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/repository/JobExecutionRepository.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package com.vmware.taurus.service;
+package com.vmware.taurus.service.repository;
 
 import com.vmware.taurus.service.model.*;
 import org.springframework.data.domain.Pageable;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/JobsRepository.java → projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/repository/JobsRepository.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package com.vmware.taurus.service;
+package com.vmware.taurus.service.repository;
 
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DeploymentStatus;

projects/control-service/projects/pipelines_control_service/src/main/resources/db/migration/V20230825140000__create_table_data_job_deployment.sql

@@ -0,0 +1,14 @@
+create table data_job_deployment
+(
+    data_job_name            varchar not null primary key references data_job (name) on delete cascade,
+    deployment_version_sha   varchar,
+    git_commit_sha           varchar,
+    python_version           varchar,
+    resources_cpu_request    float,
+    resources_cpu_limit      float,
+    resources_memory_request int,
+    resources_memory_limit   int,
+    last_deployed_date       timestamp,
+    last_deployed_by         varchar,
+    enabled                  boolean
+)

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/RepositoryUtil.java

@@ -5,8 +5,8 @@
 
 package com.vmware.taurus;
 
-import com.vmware.taurus.service.JobExecutionRepository;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobExecutionRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.model.DataJob;
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.DeploymentStatus;

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/AuthorizationInterceptorTest.java

@@ -9,7 +9,7 @@
 import com.vmware.taurus.authorization.webhook.AuthorizationWebHookProvider;
 import com.vmware.taurus.base.FeatureFlags;
 import com.vmware.taurus.security.SecurityConfiguration;
-import com.vmware.taurus.service.JobsRepository;
+import com.vmware.taurus.service.repository.JobsRepository;
 import com.vmware.taurus.service.diag.OperationContext;
 import com.vmware.taurus.service.webhook.WebHookResult;
 import org.junit.jupiter.api.Assertions;