From 04903604e8b72237fa863046fe44c7a56000dfc7 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 24 Oct 2022 15:07:52 +0100 Subject: [PATCH 01/69] vdk-pipelines-control-service: read from private repository Signed-off-by: murphp15 --- .../java/com/vmware/taurus/service/KubernetesService.java | 6 +++++- .../vmware/taurus/service/deploy/DeploymentService.java | 5 ++++- .../taurus/service/deploy/DockerRegistryService.java | 7 +++++++ .../com/vmware/taurus/service/deploy/JobImageBuilder.java | 6 ++++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 56bd1eba7c..a5c2f186df 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -1205,7 +1205,8 @@ public void createJob( long runAsUser, long runAsGroup, long fsGroup, - String serviceAccountName) + String serviceAccountName, + String registrySecret) throws ApiException { log.debug("Creating k8s job name:{}, image:{}", name, image); @@ -1231,6 +1232,9 @@ public void createJob( .runAsGroup(runAsGroup) .fsGroup(fsGroup)); + if (StringUtils.isNotEmpty(registrySecret)) { + podSpecBuilder.addNewImagePullSecretLike(new V1LocalObjectReference().name(registrySecret)); + } if (StringUtils.isNotEmpty(serviceAccountName)) { podSpecBuilder.withServiceAccountName(serviceAccountName); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java index 1ee811310d..1bde086390 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java @@ -146,7 +146,10 @@ public void updateDeployment( dockerRegistryService.dataJobImage( jobDeployment.getDataJobName(), jobDeployment.getGitCommitSha()); - if (jobImageBuilder.buildImage(imageName, dataJob, jobDeployment, sendNotification)) { + String registrySecret = dockerRegistryService.getRegistrySecret(); + + if (jobImageBuilder.buildImage(imageName, dataJob, jobDeployment, sendNotification, + registrySecret)) { log.info( "Image {} has been built. Will now schedule job {} for execution", imageName, diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java index b941277c27..707afffd9b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java @@ -17,10 +17,17 @@ public class DockerRegistryService { @Value("${datajobs.builder.image}") private String builderImage; + @Value("${datajobs.builder.registrySecret}") + private String registrySecret; + public String dataJobImage(String dataJobName, String gitCommitSha) { return String.format("%s/%s:%s", proxyRepositoryURL, dataJobName, gitCommitSha); } + public String getRegistrySecret() { + return registrySecret; + } + public String builderImage() { return builderImage; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index e6242adcee..0ed6a4960f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -121,7 +121,8 @@ public JobImageBuilder( * @throws InterruptedException */ public boolean buildImage( - String imageName, DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification) + String imageName, DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification, + String registrySecret) throws ApiException, IOException, InterruptedException { log.info("Build data job image for job {}. Image name: {}", dataJob.getName(), imageName); @@ -186,7 +187,8 @@ public boolean buildImage( builderSecurityContextRunAsUser, builderSecurityContextRunAsGroup, builderSecurityContextFsGroup, - builderServiceAccountName); + builderServiceAccountName, + registrySecret); log.debug( "Waiting for builder job {} for data job version {}", From bac776164001087b2d5d71f31f8ebb376ad31782 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 24 Oct 2022 14:08:18 +0000 Subject: [PATCH 02/69] Google Java Format --- .../vmware/taurus/service/deploy/DeploymentService.java | 4 ++-- .../com/vmware/taurus/service/deploy/JobImageBuilder.java | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java index 1bde086390..904afa5a99 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java @@ -148,8 +148,8 @@ public void updateDeployment( String registrySecret = dockerRegistryService.getRegistrySecret(); - if (jobImageBuilder.buildImage(imageName, dataJob, jobDeployment, sendNotification, - registrySecret)) { + if (jobImageBuilder.buildImage( + imageName, dataJob, jobDeployment, sendNotification, registrySecret)) { log.info( "Image {} has been built. Will now schedule job {} for execution", imageName, diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index 0ed6a4960f..d222b1aeec 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -121,7 +121,10 @@ public JobImageBuilder( * @throws InterruptedException */ public boolean buildImage( - String imageName, DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification, + String imageName, + DataJob dataJob, + JobDeployment jobDeployment, + Boolean sendNotification, String registrySecret) throws ApiException, IOException, InterruptedException { @@ -188,7 +191,7 @@ public boolean buildImage( builderSecurityContextRunAsGroup, builderSecurityContextFsGroup, builderServiceAccountName, - registrySecret); + registrySecret); log.debug( "Waiting for builder job {} for data job version {}", From 97399512f64c53a49c9335736e2b3211b8b3a88f Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 14 Nov 2022 20:39:38 +0000 Subject: [PATCH 03/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../integration-test/resources/application-test.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties index c5d9d5a549..8fef6f59b1 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties @@ -52,6 +52,10 @@ spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://test integrationTest.dataJobsNamespace=${DEPLOYMENT_K8S_NAMESPACE:} integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:} + +datajobs.builder.registrySecret=integration_test_docker_pull_secret + + datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest From 45a75cc0779a7b6e9b14f0d3fee301cc96c12d22 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 14 Nov 2022 21:07:22 +0000 Subject: [PATCH 04/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../resources/application-test.properties | 3 +-- .../taurus/datajobs/DataJobsController.java | 6 ++++++ .../service/deploy/DockerRegistryService.java | 2 +- .../vmware/taurus/datajobs/MockKubernetes.java | 2 +- .../service/deploy/DeploymentServiceTest.java | 12 ++++++------ .../service/deploy/JobImageBuilderTest.java | 18 ++++++++++-------- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties index 8fef6f59b1..37b20374b5 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties @@ -53,9 +53,8 @@ integrationTest.dataJobsNamespace=${DEPLOYMENT_K8S_NAMESPACE:} integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:} -datajobs.builder.registrySecret=integration_test_docker_pull_secret - +datajobs.builder.registrySecret=integration_test_docker_pull_secret datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java index 9cdf56f938..82b0da7e31 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java @@ -8,6 +8,7 @@ import com.vmware.taurus.controlplane.model.api.DataJobsApi; import com.vmware.taurus.controlplane.model.data.DataJob; import com.vmware.taurus.controlplane.model.data.DataJobQueryResponse; +import com.vmware.taurus.controlplane.model.data.DataJobSummary; import com.vmware.taurus.exception.ApiConstraintError; import com.vmware.taurus.exception.ExternalSystemError; import com.vmware.taurus.exception.WebHookRequestError; @@ -204,6 +205,11 @@ public ResponseEntity dataJobUpdate(String teamName, String jobName, DataJ return ResponseEntity.notFound().build(); } + @Override + public ResponseEntity> jobsList(String s, Boolean aBoolean, Integer integer, Integer integer1) { + return null; + } + @Override public ResponseEntity jobsQuery( String teamName, String query, String operationName, String variables) { diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java index 707afffd9b..d59b325a47 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java @@ -17,7 +17,7 @@ public class DockerRegistryService { @Value("${datajobs.builder.image}") private String builderImage; - @Value("${datajobs.builder.registrySecret}") + @Value("${datajobs.builder.registrySecret:}") private String registrySecret; public String dataJobImage(String dataJobName, String gitCommitSha) { diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java index 0dd4f2a6ce..84f8e78469 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java @@ -166,7 +166,7 @@ private void mockKubernetesService(KubernetesService mock) anyLong(), anyLong(), anyLong(), - anyString()); + anyString(), anyString()); doAnswer(inv -> jobs.keySet()).when(mock).listCronJobs(); doAnswer(inv -> jobs.remove(inv.getArgument(0))).when(mock).deleteJob(anyString()); diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java index e5877844e0..dbd6170d36 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java @@ -142,14 +142,14 @@ public void updateDeployment_newDeploymentCreated() when(dockerRegistryService.dataJobImage(TEST_JOB_NAME, "test-commit")) .thenReturn(TEST_JOB_IMAGE_NAME); - when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true)) + when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null)) .thenReturn(true); deploymentService.updateDeployment( testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME, OP_ID); verify(dockerRegistryService).dataJobImage(TEST_JOB_NAME, "test-commit"); - verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true); + verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null); verify(kubernetesService) .createCronJob( eq(TEST_CRONJOB_NAME), @@ -188,7 +188,7 @@ public void updateDeployment_existingDeploymentUpdated() when(dockerRegistryService.dataJobImage(TEST_JOB_NAME, "test-commit")) .thenReturn(TEST_JOB_IMAGE_NAME); - when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true)) + when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null)) .thenReturn(true); when(kubernetesService.listCronJobs()).thenReturn(Set.of(TEST_CRONJOB_NAME)); @@ -196,7 +196,7 @@ public void updateDeployment_existingDeploymentUpdated() testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME, OP_ID); verify(dockerRegistryService).dataJobImage(TEST_JOB_NAME, "test-commit"); - verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true); + verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null); verify(kubernetesService) .updateCronJob( eq(TEST_CRONJOB_NAME), @@ -232,14 +232,14 @@ public void updateDeployment_failedToBuildImage_deploymentSkipped() when(dockerRegistryService.dataJobImage(TEST_JOB_NAME, "test-commit")) .thenReturn(TEST_JOB_IMAGE_NAME); - when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true)) + when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null)) .thenReturn(false); deploymentService.updateDeployment( testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME, OP_ID); verify(dockerRegistryService).dataJobImage(TEST_JOB_NAME, "test-commit"); - verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true); + verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null); verify(kubernetesService, never()) .updateCronJob( anyString(), diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java index eb137b4f4f..83ad73c3dc 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java @@ -78,7 +78,7 @@ public void buildImage_notExist_success() throws InterruptedException, ApiExcept jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true); + var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true, null); verify(kubernetesService) .createJob( @@ -95,7 +95,7 @@ public void buildImage_notExist_success() throws InterruptedException, ApiExcept anyLong(), anyLong(), anyLong(), - any()); + any(),any()); verify(kubernetesService).deleteJob(TEST_BUILDER_JOB_NAME); Assertions.assertTrue(result); @@ -117,7 +117,7 @@ public void buildImage_builderRunning_oldBuilderDeleted() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); + var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true,null); verify(kubernetesService, times(2)).deleteJob(TEST_BUILDER_IMAGE_NAME); verify(kubernetesService) @@ -135,7 +135,7 @@ public void buildImage_builderRunning_oldBuilderDeleted() anyLong(), anyLong(), anyLong(), - any()); + any(), any()); Assertions.assertTrue(result); } @@ -149,7 +149,7 @@ public void buildImage_imageExists_buildSkipped() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); + var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true, null); verify(kubernetesService, never()) .createJob( @@ -166,7 +166,8 @@ public void buildImage_imageExists_buildSkipped() anyLong(), anyLong(), anyLong(), - anyString()); + anyString(), + anyString()); verify(notificationHelper, never()) .verifyBuilderResult(anyString(), any(), any(), any(), anyString(), anyBoolean()); Assertions.assertTrue(result); @@ -187,7 +188,7 @@ public void buildImage_jobFailed_failure() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true); + var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true, null); verify(kubernetesService) .createJob( @@ -204,7 +205,8 @@ public void buildImage_jobFailed_failure() anyLong(), anyLong(), anyLong(), - any()); + any(), + any()); // verify(kubernetesService).deleteJob(TEST_BUILDER_JOB_NAME); // not called in case of an error verify(notificationHelper) From 87c3573046909090af8c73d9f30b0d38a586fd01 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 14 Nov 2022 21:07:45 +0000 Subject: [PATCH 05/69] Google Java Format --- .../taurus/datajobs/DataJobsController.java | 3 ++- .../vmware/taurus/datajobs/MockKubernetes.java | 3 ++- .../service/deploy/JobImageBuilderTest.java | 16 ++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java index 82b0da7e31..39170f7655 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java @@ -206,7 +206,8 @@ public ResponseEntity dataJobUpdate(String teamName, String jobName, DataJ } @Override - public ResponseEntity> jobsList(String s, Boolean aBoolean, Integer integer, Integer integer1) { + public ResponseEntity> jobsList( + String s, Boolean aBoolean, Integer integer, Integer integer1) { return null; } diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java index 84f8e78469..e4b869ee4c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java @@ -166,7 +166,8 @@ private void mockKubernetesService(KubernetesService mock) anyLong(), anyLong(), anyLong(), - anyString(), anyString()); + anyString(), + anyString()); doAnswer(inv -> jobs.keySet()).when(mock).listCronJobs(); doAnswer(inv -> jobs.remove(inv.getArgument(0))).when(mock).deleteJob(anyString()); diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java index 83ad73c3dc..6f2b6debb2 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java @@ -95,7 +95,8 @@ public void buildImage_notExist_success() throws InterruptedException, ApiExcept anyLong(), anyLong(), anyLong(), - any(),any()); + any(), + any()); verify(kubernetesService).deleteJob(TEST_BUILDER_JOB_NAME); Assertions.assertTrue(result); @@ -117,7 +118,8 @@ public void buildImage_builderRunning_oldBuilderDeleted() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true,null); + var result = + jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true, null); verify(kubernetesService, times(2)).deleteJob(TEST_BUILDER_IMAGE_NAME); verify(kubernetesService) @@ -135,7 +137,8 @@ public void buildImage_builderRunning_oldBuilderDeleted() anyLong(), anyLong(), anyLong(), - any(), any()); + any(), + any()); Assertions.assertTrue(result); } @@ -149,7 +152,8 @@ public void buildImage_imageExists_buildSkipped() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true, null); + var result = + jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true, null); verify(kubernetesService, never()) .createJob( @@ -167,7 +171,7 @@ public void buildImage_imageExists_buildSkipped() anyLong(), anyLong(), anyString(), - anyString()); + anyString()); verify(notificationHelper, never()) .verifyBuilderResult(anyString(), any(), any(), any(), anyString(), anyBoolean()); Assertions.assertTrue(result); @@ -206,7 +210,7 @@ public void buildImage_jobFailed_failure() anyLong(), anyLong(), any(), - any()); + any()); // verify(kubernetesService).deleteJob(TEST_BUILDER_JOB_NAME); // not called in case of an error verify(notificationHelper) From 71d3bbd756a2c3422275a34d05199bbcb85310b2 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 11:34:01 +0000 Subject: [PATCH 06/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/datajobs/it/common/BaseIT.java | 19 +++++++++++++++++++ .../resources/application-test.properties | 4 ++-- .../taurus/service/KubernetesService.java | 9 +++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 7b1363fad1..05c6be1a2a 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -17,6 +17,8 @@ import com.vmware.taurus.service.kubernetes.DataJobsKubernetesService; import com.vmware.taurus.service.model.JobConfig; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1SecretBuilder; import org.apache.commons.lang3.StringUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; @@ -42,6 +44,8 @@ import java.time.Instant; import java.time.OffsetDateTime; +import java.util.HashMap; +import java.util.Map; import java.util.function.Predicate; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; @@ -94,6 +98,20 @@ public KerberosCredentialsRepository credentialsRepository() { private boolean ownsControlNamespace = false; + + public void createBuilderImagePullSecret(String namespaceName) throws ApiException, JsonProcessingException { + new CoreV1Api(controlKubernetesService.getClient()).createNamespacedSecret(namespaceName, new V1SecretBuilder() + .withNewMetadata() + .withName("integration-test-docker-pull-secret") + .withNamespace(namespaceName) + .endMetadata() + .withStringData(Map.of(".dockerconfigjson", new ObjectMapper().writeValueAsString(Map.of("auths", Map.of("vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + Map.of("auth", "c3ZjLnRhdXItamZyb2ctcnc6QUtDcDhrcU1UTDF6ckRQY044OGVNeG5WdHBreXVRR0pXNVNjMXZkc0prUVFKQ29xUEcyZVpzOVJpNmozczVNNzNRR3Bjb01tdQ==")))))) + .withType("kubernetes.io/dockerconfigjson") + .build(), null ,null ,null ,null); + } + + @BeforeEach public void before() throws Exception { log.info("Running test with: {} bytes of memory.", Runtime.getRuntime().totalMemory()); @@ -116,6 +134,7 @@ public void before() throws Exception { ; log.info("Create namespace {}", controlNamespace); controlKubernetesService.createNamespace(controlNamespace); + createBuilderImagePullSecret(controlNamespace); this.ownsControlNamespace = true; } else { log.info("Using predefined control namespace {}", controlNamespace); diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties index 37b20374b5..4dd51d02cb 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties @@ -54,8 +54,8 @@ integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:} -datajobs.builder.registrySecret=integration_test_docker_pull_secret -datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 +datajobs.builder.registrySecret=integration-test-docker-pull-secret +datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 1a5a9905d2..e54cdf2e9b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -1235,7 +1235,7 @@ public void createJob( .fsGroup(fsGroup)); if (StringUtils.isNotEmpty(registrySecret)) { - podSpecBuilder.addNewImagePullSecretLike(new V1LocalObjectReference().name(registrySecret)); + podSpecBuilder.addNewImagePullSecret().withName(registrySecret).endImagePullSecret(); } if (StringUtils.isNotEmpty(serviceAccountName)) { podSpecBuilder.withServiceAccountName(serviceAccountName); @@ -1249,7 +1249,7 @@ public void createJob( .withTemplate(template) .build(); createNewJob(name, spec, Collections.emptyMap(), Collections.emptyMap()); - } + } // Default for testing purposes void createNewJob( @@ -2504,4 +2504,9 @@ static int convertMemoryToMBs(Quantity quantity) { } return quantity.getNumber().toBigInteger().divide(divider.multiply(divider)).intValue(); } + + + public ApiClient getClient() { + return client; + } } From f08e63318eb016214bdc1cae3542ddb16c967964 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 11:34:26 +0000 Subject: [PATCH 07/69] Google Java Format --- .../taurus/datajobs/it/common/BaseIT.java | 41 +++++++++++++------ .../taurus/service/KubernetesService.java | 3 +- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 05c6be1a2a..eaa1e3fdd4 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -44,7 +44,6 @@ import java.time.Instant; import java.time.OffsetDateTime; -import java.util.HashMap; import java.util.Map; import java.util.function.Predicate; @@ -98,20 +97,36 @@ public KerberosCredentialsRepository credentialsRepository() { private boolean ownsControlNamespace = false; - - public void createBuilderImagePullSecret(String namespaceName) throws ApiException, JsonProcessingException { - new CoreV1Api(controlKubernetesService.getClient()).createNamespacedSecret(namespaceName, new V1SecretBuilder() - .withNewMetadata() - .withName("integration-test-docker-pull-secret") - .withNamespace(namespaceName) - .endMetadata() - .withStringData(Map.of(".dockerconfigjson", new ObjectMapper().writeValueAsString(Map.of("auths", Map.of("vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - Map.of("auth", "c3ZjLnRhdXItamZyb2ctcnc6QUtDcDhrcU1UTDF6ckRQY044OGVNeG5WdHBreXVRR0pXNVNjMXZkc0prUVFKQ29xUEcyZVpzOVJpNmozczVNNzNRR3Bjb01tdQ==")))))) - .withType("kubernetes.io/dockerconfigjson") - .build(), null ,null ,null ,null); + public void createBuilderImagePullSecret(String namespaceName) + throws ApiException, JsonProcessingException { + new CoreV1Api(controlKubernetesService.getClient()) + .createNamespacedSecret( + namespaceName, + new V1SecretBuilder() + .withNewMetadata() + .withName("integration-test-docker-pull-secret") + .withNamespace(namespaceName) + .endMetadata() + .withStringData( + Map.of( + ".dockerconfigjson", + new ObjectMapper() + .writeValueAsString( + Map.of( + "auths", + Map.of( + "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + Map.of( + "auth", + "c3ZjLnRhdXItamZyb2ctcnc6QUtDcDhrcU1UTDF6ckRQY044OGVNeG5WdHBreXVRR0pXNVNjMXZkc0prUVFKQ29xUEcyZVpzOVJpNmozczVNNzNRR3Bjb01tdQ==")))))) + .withType("kubernetes.io/dockerconfigjson") + .build(), + null, + null, + null, + null); } - @BeforeEach public void before() throws Exception { log.info("Running test with: {} bytes of memory.", Runtime.getRuntime().totalMemory()); diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index e54cdf2e9b..3fcc044692 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -1249,7 +1249,7 @@ public void createJob( .withTemplate(template) .build(); createNewJob(name, spec, Collections.emptyMap(), Collections.emptyMap()); - } + } // Default for testing purposes void createNewJob( @@ -2505,7 +2505,6 @@ static int convertMemoryToMBs(Quantity quantity) { return quantity.getNumber().toBigInteger().divide(divider.multiply(divider)).intValue(); } - public ApiClient getClient() { return client; } From 151591103b911b27852b92e04afe50a4deab53f2 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 11:34:01 +0000 Subject: [PATCH 08/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../java/com/vmware/taurus/datajobs/it/common/BaseIT.java | 2 +- .../main/java/com/vmware/taurus/service/KubernetesService.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index eaa1e3fdd4..9a71e7d0f1 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -118,7 +118,7 @@ public void createBuilderImagePullSecret(String namespaceName) "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", Map.of( "auth", - "c3ZjLnRhdXItamZyb2ctcnc6QUtDcDhrcU1UTDF6ckRQY044OGVNeG5WdHBreXVRR0pXNVNjMXZkc0prUVFKQ29xUEcyZVpzOVJpNmozczVNNzNRR3Bjb01tdQ==")))))) + "")))))) .withType("kubernetes.io/dockerconfigjson") .build(), null, diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 3fcc044692..e54cdf2e9b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -1249,7 +1249,7 @@ public void createJob( .withTemplate(template) .build(); createNewJob(name, spec, Collections.emptyMap(), Collections.emptyMap()); - } + } // Default for testing purposes void createNewJob( @@ -2505,6 +2505,7 @@ static int convertMemoryToMBs(Quantity quantity) { return quantity.getNumber().toBigInteger().divide(divider.multiply(divider)).intValue(); } + public ApiClient getClient() { return client; } From 375013e94d9aabba02f3d5cd96a4d52d771e94e0 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 11:37:34 +0000 Subject: [PATCH 09/69] Google Java Format --- .../java/com/vmware/taurus/datajobs/it/common/BaseIT.java | 4 +--- .../java/com/vmware/taurus/service/KubernetesService.java | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 9a71e7d0f1..b1896a8ddf 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -116,9 +116,7 @@ public void createBuilderImagePullSecret(String namespaceName) "auths", Map.of( "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - Map.of( - "auth", - "")))))) + Map.of("auth", "")))))) .withType("kubernetes.io/dockerconfigjson") .build(), null, diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index e54cdf2e9b..3fcc044692 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -1249,7 +1249,7 @@ public void createJob( .withTemplate(template) .build(); createNewJob(name, spec, Collections.emptyMap(), Collections.emptyMap()); - } + } // Default for testing purposes void createNewJob( @@ -2505,7 +2505,6 @@ static int convertMemoryToMBs(Quantity quantity) { return quantity.getNumber().toBigInteger().divide(divider.multiply(divider)).intValue(); } - public ApiClient getClient() { return client; } From 5520601f4e4b3039244d65cb7fb827e426c42f84 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 12:33:54 +0000 Subject: [PATCH 10/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../it/PrivateBuilderDockerRepoIT.java | 413 ++++++++++++++++++ .../taurus/datajobs/it/common/BaseIT.java | 30 +- .../application-private-builder.properties | 3 + .../resources/application-test.properties | 3 +- 4 files changed, 419 insertions(+), 30 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java new file mode 100644 index 0000000000..f1a24629c7 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -0,0 +1,413 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.controlplane.model.data.DataJobExecution; +import com.vmware.taurus.controlplane.model.data.DataJobVersion; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1SecretBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MvcResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; +import static org.awaitility.Awaitility.await; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@Slf4j +@Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class}) +@TestPropertySource( + properties = { + "dataJob.readOnlyRootFileSystem=true", + }) +@ActiveProfiles({"test", "private-builder"}) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class PrivateBuilderDockerRepoIT extends BaseIT { + + private static final String TEST_JOB_NAME = + "ephemeral-storage-test-" + UUID.randomUUID().toString().substring(0, 8); + private static final Object DEPLOYMENT_ID = "testing-ephemeral-storage"; + private final ObjectMapper objectMapper = + new ObjectMapper() + .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; + + @Value("${datajobs.builder.registrySecret.content.testOnly:}") + private String dataJobsBuilderRegistrySecretContent; + + + public void createBuilderImagePullSecret(String namespaceName) + throws ApiException, JsonProcessingException { + new CoreV1Api(controlKubernetesService.getClient()) + .createNamespacedSecret( + namespaceName, + new V1SecretBuilder() + .withNewMetadata() + .withName("integration-test-docker-pull-secret") + .withNamespace(namespaceName) + .endMetadata() + .withStringData( + Map.of( + ".dockerconfigjson", + new ObjectMapper() + .writeValueAsString( + Map.of( + "auths", + Map.of( + "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + Map.of( + "auth", + dataJobsBuilderRegistrySecretContent)))))) + .withType("kubernetes.io/dockerconfigjson") + .build(), + null, + null, + null, + null); + } + + + @AfterEach + public void cleanUp() throws Exception { + // delete job + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isOk()); + + // Execute delete deployment + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + } + + @BeforeEach + public void setup() throws Exception { + String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + // Execute create job + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()); + } + + @Test + public void testPrivateDockerBuildJob() throws Exception { + createBuilderImagePullSecret(getControlNamespace()); + // Take the job zip as byte array + byte[] jobZipBinary = + IOUtils.toByteArray( + getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); + + // Execute job upload with user + MvcResult jobUploadResult = + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(jobZipBinary) + .contentType(MediaType.APPLICATION_OCTET_STREAM)) + .andExpect(status().isOk()) + .andReturn(); + + DataJobVersion testDataJobVersion = + new ObjectMapper() + .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); + Assertions.assertNotNull(testDataJobVersion); + + String testJobVersionSha = testDataJobVersion.getVersionSha(); + Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); + + // Setup + String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); + + // Execute build and deploy job + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(dataJobDeploymentRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()) + .andReturn(); + + String opId = TEST_JOB_NAME + UUID.randomUUID().toString().toLowerCase(); + + // manually start job execution + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .header(HEADER_X_OP_ID, opId) + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\n" + + " \"args\": {\n" + + " \"key\": \"value\"\n" + + " },\n" + + " \"started_by\": \"schedule/runtime\"\n" + + "}")) + .andExpect(status().is(202)) + .andReturn(); + + // wait for pod to initialize + Thread.sleep(10000); + + // retrieve running job execution id. + var exc = + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + var gson = new Gson(); + ArrayList parsed = + gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); + String executionId = (String) parsed.get(0).get("id"); + + // Check the data job execution status + checkDataJobExecutionStatus( + executionId, + DataJobExecution.StatusEnum.SUCCEEDED, + opId, + TEST_JOB_NAME, + TEST_TEAM_NAME, + "user"); + } + + private void checkDataJobExecutionStatus( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + try { + testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobDeploymentExecutionList( + executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionLogs(executionId, jobName, teamName, username); + } catch (Error e) { + try { + // print logs in case execution has failed + MvcResult dataJobExecutionLogsResult = + getExecuteLogs(executionId, jobName, teamName, username); + log.info( + "Job Execution {} logs:\n{}", + executionId, + dataJobExecutionLogsResult.getResponse().getContentAsString()); + } catch (Error ignore) { + } + throw e; + } + } + + private void testDataJobExecutionRead( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) { + + DataJobExecution[] dataJobExecution = new DataJobExecution[1]; + + await() + .atMost(5, TimeUnit.MINUTES) + .with() + .pollInterval(15, TimeUnit.SECONDS) + .until( + () -> { + String dataJobExecutionReadUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s", + teamName, jobName, executionId); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionReadUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + dataJobExecution[0] = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), + DataJobExecution.class); + if (dataJobExecution[0] == null) { + log.info("No response from server"); + } else { + log.info("Response from server " + dataJobExecution[0].getStatus()); + } + return dataJobExecution[0] != null + && executionStatus.equals(dataJobExecution[0].getStatus()); + }); + + assertDataJobExecutionValid(executionId, executionStatus, opId, dataJobExecution[0], jobName); + } + + private void testDataJobExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobExecutionListUrl = + String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() { + }); + Assertions.assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + Assertions.assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); + } + + private void testDataJobDeploymentExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobDeploymentExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + teamName, jobName, "release"); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobDeploymentExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() { + }); + Assertions.assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + Assertions.assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); + } + + private void testDataJobExecutionLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); + Assertions.assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); + } + + @NotNull + private MvcResult getExecuteLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + String dataJobExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); + MvcResult dataJobExecutionLogsResult = + mockMvc + .perform(get(dataJobExecutionListUrl).with(user(username))) + .andExpect(status().isOk()) + .andReturn(); + return dataJobExecutionLogsResult; + } + + private void assertDataJobExecutionValid( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + DataJobExecution dataJobExecution, + String jobName) { + + Assertions.assertNotNull(dataJobExecution); + Assertions.assertEquals(executionId, dataJobExecution.getId()); + Assertions.assertEquals(jobName, dataJobExecution.getJobName()); + Assertions.assertEquals(executionStatus, dataJobExecution.getStatus()); + Assertions.assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); + Assertions.assertEquals(opId, dataJobExecution.getOpId()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index b1896a8ddf..6a9e2bd27b 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -94,35 +94,10 @@ public KerberosCredentialsRepository credentialsRepository() { @Value("${integrationTest.controlNamespace:}") private String controlNamespace; - private boolean ownsControlNamespace = false; - public void createBuilderImagePullSecret(String namespaceName) - throws ApiException, JsonProcessingException { - new CoreV1Api(controlKubernetesService.getClient()) - .createNamespacedSecret( - namespaceName, - new V1SecretBuilder() - .withNewMetadata() - .withName("integration-test-docker-pull-secret") - .withNamespace(namespaceName) - .endMetadata() - .withStringData( - Map.of( - ".dockerconfigjson", - new ObjectMapper() - .writeValueAsString( - Map.of( - "auths", - Map.of( - "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - Map.of("auth", "")))))) - .withType("kubernetes.io/dockerconfigjson") - .build(), - null, - null, - null, - null); + public String getControlNamespace() { + return controlNamespace; } @BeforeEach @@ -147,7 +122,6 @@ public void before() throws Exception { ; log.info("Create namespace {}", controlNamespace); controlKubernetesService.createNamespace(controlNamespace); - createBuilderImagePullSecret(controlNamespace); this.ownsControlNamespace = true; } else { log.info("Using predefined control namespace {}", controlNamespace); diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties new file mode 100644 index 0000000000..2a61b3c4ab --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -0,0 +1,3 @@ +datajobs.builder.registrySecret=integration-test-docker-pull-secret +datajobs.builder.registrySecret.content.testOnly={BUILDER_TEST_REGISTRY_SECRET} +datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties index 4dd51d02cb..aba48032ec 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties @@ -54,8 +54,7 @@ integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:} -datajobs.builder.registrySecret=integration-test-docker-pull-secret -datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 +datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest From 0a7cb1ce06863faba1c26bca8a017b2bd456595a Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 12:37:46 +0000 Subject: [PATCH 11/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../resources/application-private-builder.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties index 2a61b3c4ab..d643b677c3 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -1,3 +1,5 @@ datajobs.builder.registrySecret=integration-test-docker-pull-secret datajobs.builder.registrySecret.content.testOnly={BUILDER_TEST_REGISTRY_SECRET} datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 + + From 96424a6284500c12cfab709a22c0cba016d03938 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 12:41:53 +0000 Subject: [PATCH 12/69] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../resources/application-private-builder.properties | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties index d643b677c3..2a61b3c4ab 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -1,5 +1,3 @@ datajobs.builder.registrySecret=integration-test-docker-pull-secret datajobs.builder.registrySecret.content.testOnly={BUILDER_TEST_REGISTRY_SECRET} datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 - - From 262c5cd57b7355bd5cabf6dca353d8f2f88ba6fb Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 12:52:18 +0000 Subject: [PATCH 13/69] Google Java Format --- .../it/PrivateBuilderDockerRepoIT.java | 694 +++++++++--------- .../taurus/datajobs/it/common/BaseIT.java | 4 +- 2 files changed, 345 insertions(+), 353 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index f1a24629c7..30f87ff978 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -50,364 +50,358 @@ @Slf4j @Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class}) @TestPropertySource( - properties = { - "dataJob.readOnlyRootFileSystem=true", - }) + properties = { + "dataJob.readOnlyRootFileSystem=true", + }) @ActiveProfiles({"test", "private-builder"}) @SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) public class PrivateBuilderDockerRepoIT extends BaseIT { - private static final String TEST_JOB_NAME = - "ephemeral-storage-test-" + UUID.randomUUID().toString().substring(0, 8); - private static final Object DEPLOYMENT_ID = "testing-ephemeral-storage"; - private final ObjectMapper objectMapper = - new ObjectMapper() - .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; - - @Value("${datajobs.builder.registrySecret.content.testOnly:}") - private String dataJobsBuilderRegistrySecretContent; - - - public void createBuilderImagePullSecret(String namespaceName) - throws ApiException, JsonProcessingException { - new CoreV1Api(controlKubernetesService.getClient()) - .createNamespacedSecret( - namespaceName, - new V1SecretBuilder() - .withNewMetadata() - .withName("integration-test-docker-pull-secret") - .withNamespace(namespaceName) - .endMetadata() - .withStringData( - Map.of( - ".dockerconfigjson", - new ObjectMapper() - .writeValueAsString( - Map.of( - "auths", - Map.of( - "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - Map.of( - "auth", - dataJobsBuilderRegistrySecretContent)))))) - .withType("kubernetes.io/dockerconfigjson") - .build(), - null, - null, - null, - null); - } - - - @AfterEach - public void cleanUp() throws Exception { - // delete job + private static final String TEST_JOB_NAME = + "ephemeral-storage-test-" + UUID.randomUUID().toString().substring(0, 8); + private static final Object DEPLOYMENT_ID = "testing-ephemeral-storage"; + private final ObjectMapper objectMapper = + new ObjectMapper() + .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; + + @Value("${datajobs.builder.registrySecret.content.testOnly:}") + private String dataJobsBuilderRegistrySecretContent; + + public void createBuilderImagePullSecret(String namespaceName) + throws ApiException, JsonProcessingException { + new CoreV1Api(controlKubernetesService.getClient()) + .createNamespacedSecret( + namespaceName, + new V1SecretBuilder() + .withNewMetadata() + .withName("integration-test-docker-pull-secret") + .withNamespace(namespaceName) + .endMetadata() + .withStringData( + Map.of( + ".dockerconfigjson", + new ObjectMapper() + .writeValueAsString( + Map.of( + "auths", + Map.of( + "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + Map.of("auth", dataJobsBuilderRegistrySecretContent)))))) + .withType("kubernetes.io/dockerconfigjson") + .build(), + null, + null, + null, + null); + } + + @AfterEach + public void cleanUp() throws Exception { + // delete job + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isOk()); + + // Execute delete deployment + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + } + + @BeforeEach + public void setup() throws Exception { + String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + // Execute create job + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()); + } + + @Test + public void testPrivateDockerBuildJob() throws Exception { + createBuilderImagePullSecret(getControlNamespace()); + // Take the job zip as byte array + byte[] jobZipBinary = + IOUtils.toByteArray( + getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); + + // Execute job upload with user + MvcResult jobUploadResult = mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user"))) - .andExpect(status().isOk()); - - // Execute delete deployment + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(jobZipBinary) + .contentType(MediaType.APPLICATION_OCTET_STREAM)) + .andExpect(status().isOk()) + .andReturn(); + + DataJobVersion testDataJobVersion = + new ObjectMapper() + .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); + Assertions.assertNotNull(testDataJobVersion); + + String testJobVersionSha = testDataJobVersion.getVersionSha(); + Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); + + // Setup + String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); + + // Execute build and deploy job + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(dataJobDeploymentRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()) + .andReturn(); + + String opId = TEST_JOB_NAME + UUID.randomUUID().toString().toLowerCase(); + + // manually start job execution + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .header(HEADER_X_OP_ID, opId) + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\n" + + " \"args\": {\n" + + " \"key\": \"value\"\n" + + " },\n" + + " \"started_by\": \"schedule/runtime\"\n" + + "}")) + .andExpect(status().is(202)) + .andReturn(); + + // wait for pod to initialize + Thread.sleep(10000); + + // retrieve running job execution id. + var exc = mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()); + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + var gson = new Gson(); + ArrayList parsed = + gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); + String executionId = (String) parsed.get(0).get("id"); + + // Check the data job execution status + checkDataJobExecutionStatus( + executionId, + DataJobExecution.StatusEnum.SUCCEEDED, + opId, + TEST_JOB_NAME, + TEST_TEAM_NAME, + "user"); + } + + private void checkDataJobExecutionStatus( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + try { + testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobDeploymentExecutionList( + executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionLogs(executionId, jobName, teamName, username); + } catch (Error e) { + try { + // print logs in case execution has failed + MvcResult dataJobExecutionLogsResult = + getExecuteLogs(executionId, jobName, teamName, username); + log.info( + "Job Execution {} logs:\n{}", + executionId, + dataJobExecutionLogsResult.getResponse().getContentAsString()); + } catch (Error ignore) { + } + throw e; } - - @BeforeEach - public void setup() throws Exception { - String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); - - // Execute create job + } + + private void testDataJobExecutionRead( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) { + + DataJobExecution[] dataJobExecution = new DataJobExecution[1]; + + await() + .atMost(5, TimeUnit.MINUTES) + .with() + .pollInterval(15, TimeUnit.SECONDS) + .until( + () -> { + String dataJobExecutionReadUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s", + teamName, jobName, executionId); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionReadUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + dataJobExecution[0] = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), + DataJobExecution.class); + if (dataJobExecution[0] == null) { + log.info("No response from server"); + } else { + log.info("Response from server " + dataJobExecution[0].getStatus()); + } + return dataJobExecution[0] != null + && executionStatus.equals(dataJobExecution[0].getStatus()); + }); + + assertDataJobExecutionValid(executionId, executionStatus, opId, dataJobExecution[0], jobName); + } + + private void testDataJobExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobExecutionListUrl = + String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); + MvcResult dataJobExecutionResult = mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()); - } - - @Test - public void testPrivateDockerBuildJob() throws Exception { - createBuilderImagePullSecret(getControlNamespace()); - // Take the job zip as byte array - byte[] jobZipBinary = - IOUtils.toByteArray( - getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); - - // Execute job upload with user - MvcResult jobUploadResult = - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .content(jobZipBinary) - .contentType(MediaType.APPLICATION_OCTET_STREAM)) - .andExpect(status().isOk()) - .andReturn(); - - DataJobVersion testDataJobVersion = - new ObjectMapper() - .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); - Assertions.assertNotNull(testDataJobVersion); - - String testJobVersionSha = testDataJobVersion.getVersionSha(); - Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); - - // Setup - String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); - - // Execute build and deploy job + .perform( + get(dataJobExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); + Assertions.assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + Assertions.assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); + } + + private void testDataJobDeploymentExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobDeploymentExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + teamName, jobName, "release"); + MvcResult dataJobExecutionResult = mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .content(dataJobDeploymentRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()) - .andReturn(); - - String opId = TEST_JOB_NAME + UUID.randomUUID().toString().toLowerCase(); - - // manually start job execution + .perform( + get(dataJobDeploymentExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); + Assertions.assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + Assertions.assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); + } + + private void testDataJobExecutionLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); + Assertions.assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); + } + + @NotNull + private MvcResult getExecuteLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + String dataJobExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); + MvcResult dataJobExecutionLogsResult = mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .header(HEADER_X_OP_ID, opId) - .contentType(MediaType.APPLICATION_JSON) - .content( - "{\n" - + " \"args\": {\n" - + " \"key\": \"value\"\n" - + " },\n" - + " \"started_by\": \"schedule/runtime\"\n" - + "}")) - .andExpect(status().is(202)) - .andReturn(); - - // wait for pod to initialize - Thread.sleep(10000); - - // retrieve running job execution id. - var exc = - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - var gson = new Gson(); - ArrayList parsed = - gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); - String executionId = (String) parsed.get(0).get("id"); - - // Check the data job execution status - checkDataJobExecutionStatus( - executionId, - DataJobExecution.StatusEnum.SUCCEEDED, - opId, - TEST_JOB_NAME, - TEST_TEAM_NAME, - "user"); - } - - private void checkDataJobExecutionStatus( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - try { - testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobDeploymentExecutionList( - executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionLogs(executionId, jobName, teamName, username); - } catch (Error e) { - try { - // print logs in case execution has failed - MvcResult dataJobExecutionLogsResult = - getExecuteLogs(executionId, jobName, teamName, username); - log.info( - "Job Execution {} logs:\n{}", - executionId, - dataJobExecutionLogsResult.getResponse().getContentAsString()); - } catch (Error ignore) { - } - throw e; - } - } - - private void testDataJobExecutionRead( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) { - - DataJobExecution[] dataJobExecution = new DataJobExecution[1]; - - await() - .atMost(5, TimeUnit.MINUTES) - .with() - .pollInterval(15, TimeUnit.SECONDS) - .until( - () -> { - String dataJobExecutionReadUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s", - teamName, jobName, executionId); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionReadUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - dataJobExecution[0] = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), - DataJobExecution.class); - if (dataJobExecution[0] == null) { - log.info("No response from server"); - } else { - log.info("Response from server " + dataJobExecution[0].getStatus()); - } - return dataJobExecution[0] != null - && executionStatus.equals(dataJobExecution[0].getStatus()); - }); - - assertDataJobExecutionValid(executionId, executionStatus, opId, dataJobExecution[0], jobName); - } - - private void testDataJobExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobExecutionListUrl = - String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() { - }); - Assertions.assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - Assertions.assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); - } - - private void testDataJobDeploymentExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobDeploymentExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - teamName, jobName, "release"); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobDeploymentExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() { - }); - Assertions.assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - Assertions.assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); - } - - private void testDataJobExecutionLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); - Assertions.assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); - } - - @NotNull - private MvcResult getExecuteLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - String dataJobExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); - MvcResult dataJobExecutionLogsResult = - mockMvc - .perform(get(dataJobExecutionListUrl).with(user(username))) - .andExpect(status().isOk()) - .andReturn(); - return dataJobExecutionLogsResult; - } - - private void assertDataJobExecutionValid( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - DataJobExecution dataJobExecution, - String jobName) { - - Assertions.assertNotNull(dataJobExecution); - Assertions.assertEquals(executionId, dataJobExecution.getId()); - Assertions.assertEquals(jobName, dataJobExecution.getJobName()); - Assertions.assertEquals(executionStatus, dataJobExecution.getStatus()); - Assertions.assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); - Assertions.assertEquals(opId, dataJobExecution.getOpId()); - } + .perform(get(dataJobExecutionListUrl).with(user(username))) + .andExpect(status().isOk()) + .andReturn(); + return dataJobExecutionLogsResult; + } + + private void assertDataJobExecutionValid( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + DataJobExecution dataJobExecution, + String jobName) { + + Assertions.assertNotNull(dataJobExecution); + Assertions.assertEquals(executionId, dataJobExecution.getId()); + Assertions.assertEquals(jobName, dataJobExecution.getJobName()); + Assertions.assertEquals(executionStatus, dataJobExecution.getStatus()); + Assertions.assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); + Assertions.assertEquals(opId, dataJobExecution.getOpId()); + } } diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 6a9e2bd27b..2e340372fc 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -17,8 +17,6 @@ import com.vmware.taurus.service.kubernetes.DataJobsKubernetesService; import com.vmware.taurus.service.model.JobConfig; import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.apis.CoreV1Api; -import io.kubernetes.client.openapi.models.V1SecretBuilder; import org.apache.commons.lang3.StringUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; @@ -44,7 +42,6 @@ import java.time.Instant; import java.time.OffsetDateTime; -import java.util.Map; import java.util.function.Predicate; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; @@ -94,6 +91,7 @@ public KerberosCredentialsRepository credentialsRepository() { @Value("${integrationTest.controlNamespace:}") private String controlNamespace; + private boolean ownsControlNamespace = false; public String getControlNamespace() { From ec9988cc8781e2cf3a21d8b3df4f2bfcff5563f6 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 12:56:31 +0000 Subject: [PATCH 14/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/authorization/TestKerberosServerHelper.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java index 9c6f4efb1a..50daef77a2 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java @@ -61,6 +61,10 @@ public class TestKerberosServerHelper { public static void shutdownServer() throws KrbException, IOException { simpleKdcServer.stop(); Files.delete(KEYTAB.toPath()); - Files.delete(KDC_WORK_DIR.toPath()); + try { + Files.delete(KDC_WORK_DIR.toPath()); + }catch(Exception e){ + log.error("Failed to delete test directory.", e); + } } } From 9229b44229647903307a0fac20702847dd7d4139 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 12:57:00 +0000 Subject: [PATCH 15/69] Google Java Format --- .../vmware/taurus/authorization/TestKerberosServerHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java index 50daef77a2..454f644d90 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java @@ -63,7 +63,7 @@ public static void shutdownServer() throws KrbException, IOException { Files.delete(KEYTAB.toPath()); try { Files.delete(KDC_WORK_DIR.toPath()); - }catch(Exception e){ + } catch (Exception e) { log.error("Failed to delete test directory.", e); } } From 5863b813b01e1f1b87a25cb6f803e6d57457cf3f Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 13:16:10 +0000 Subject: [PATCH 16/69] vdk-control-service: set registry name correctly. Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 701bf17a01..ba071ea3c6 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -48,10 +48,9 @@ control_service_build_image: - apk --no-cache add git openjdk11-jdk --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community - export TAG=$(git rev-parse --short HEAD) - cd projects/control-service/projects - - ./gradlew -p ./model build publishToMavenLocal --info --stacktrace - - ./gradlew build jacocoTestReport -x integrationTest --info --stacktrace - - ./gradlew :pipelines_control_service:docker --info --stacktrace -Pversion=$TAG - retry: !reference [.control_service_retry, retry_options] + - ./gradlew -p ./model build publishToMavenLocal + - ./gradlew build jacocoTestReport -x integrationTest + retry: !reference [.control_service_retry] coverage: "/ - Line Coverage: ([0-9.]+)%/" artifacts: when: always From 63d84e6196110b99eb882c378d7c9efb13a6beec Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 13:19:05 +0000 Subject: [PATCH 17/69] vdk-control-service: set registry name correctly. Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index ba071ea3c6..d1cbc10161 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -50,7 +50,6 @@ control_service_build_image: - cd projects/control-service/projects - ./gradlew -p ./model build publishToMavenLocal - ./gradlew build jacocoTestReport -x integrationTest - retry: !reference [.control_service_retry] coverage: "/ - Line Coverage: ([0-9.]+)%/" artifacts: when: always From b00af22f395493f72cead94691eb30d9a7f5d681 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 13:22:13 +0000 Subject: [PATCH 18/69] vdk-control-service: set registry name correctly. Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index d1cbc10161..64d1d616c9 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -64,6 +64,8 @@ control_service_build_image: - main changes: *control_service_change_locations + + control_service_integration_test: extends: .control_service_base_build stage: build From 3e807024ea7ba3ea86847b271cb158a09abd0aaa Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 13:29:04 +0000 Subject: [PATCH 19/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../java/com/vmware/taurus/datajobs/DataJobsController.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java index 39170f7655..b5da0e55d7 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java @@ -205,12 +205,6 @@ public ResponseEntity dataJobUpdate(String teamName, String jobName, DataJ return ResponseEntity.notFound().build(); } - @Override - public ResponseEntity> jobsList( - String s, Boolean aBoolean, Integer integer, Integer integer1) { - return null; - } - @Override public ResponseEntity jobsQuery( String teamName, String query, String operationName, String variables) { From 0be61ccb150db321cbfb7e5f1e497010fcb232d6 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 13:31:38 +0000 Subject: [PATCH 20/69] Google Java Format --- .../main/java/com/vmware/taurus/datajobs/DataJobsController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java index b5da0e55d7..9cdf56f938 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsController.java @@ -8,7 +8,6 @@ import com.vmware.taurus.controlplane.model.api.DataJobsApi; import com.vmware.taurus.controlplane.model.data.DataJob; import com.vmware.taurus.controlplane.model.data.DataJobQueryResponse; -import com.vmware.taurus.controlplane.model.data.DataJobSummary; import com.vmware.taurus.exception.ApiConstraintError; import com.vmware.taurus.exception.ExternalSystemError; import com.vmware.taurus.exception.WebHookRequestError; From 93b9649e2394e51538570ae60dfb0d3626b00ab3 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 15:39:26 +0000 Subject: [PATCH 21/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 30f87ff978..7ccd65c807 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -60,8 +60,8 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { private static final String TEST_JOB_NAME = - "ephemeral-storage-test-" + UUID.randomUUID().toString().substring(0, 8); - private static final Object DEPLOYMENT_ID = "testing-ephemeral-storage"; + "private-docker-builder-test-" + UUID.randomUUID().toString().substring(0, 8); + private static final Object DEPLOYMENT_ID = "private-docker-builder"; private final ObjectMapper objectMapper = new ObjectMapper() .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; From da958b1c38fd3ce1e623ba14aa1c5b774982f288 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 15:42:28 +0000 Subject: [PATCH 22/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../datajobs/it/DataJobCancellationIT.java | 200 -------- .../taurus/datajobs/it/DataJobCrudIT.java | 460 ------------------ .../it/DataJobEphemeralStorageIT.java | 369 -------------- .../taurus/datajobs/it/DataJobGraphQLIT.java | 293 ----------- .../datajobs/it/DataJobPropertiesIT.java | 88 ---- .../it/DataJobTerminationStatusIT.java | 386 --------------- .../it/common/BaseDataJobDeploymentIT.java | 88 ---- .../datajobs/it/common/JobExecutionUtil.java | 49 -- .../KerberosSecurityTestcaseJunit5.java | 25 - .../common/MiniKdcCredentialsRepository.java | 53 -- .../graphql/it/GraphQLDataJobsFieldsIT.java | 91 ---- .../it/GraphQLDataJobsSortContactsIT.java | 190 -------- .../graphql/it/GraphQLExecutionsIT.java | 430 ---------------- .../it/GraphQLExecutionsLogsUrlIT.java | 179 ------- .../it/GraphQLExecutionsNextRunIT.java | 190 -------- .../GraphQLJobExecutionsSortByEndTimeIT.java | 203 -------- .../it/GraphQLJobExecutionsStatusCountIT.java | 174 ------- 17 files changed, 3468 deletions(-) delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java delete mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java deleted file mode 100644 index 4ac41b3390..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import com.google.gson.internal.LinkedTreeMap; -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.controlplane.model.data.DataJobVersion; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.platform.commons.util.StringUtils; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Import; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MvcResult; - -import java.util.ArrayList; -import java.util.UUID; - -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@Slf4j -@Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class}) -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class DataJobCancellationIT extends BaseIT { - - private static final String TEST_JOB_NAME = - "cancellation-test-" + UUID.randomUUID().toString().substring(0, 8); - private static final Object DEPLOYMENT_ID = "testing-cancellation"; - - @AfterEach - public void cleanUp() throws Exception { - // delete job - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user"))) - .andExpect(status().isOk()); - - // Execute delete deployment - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()); - } - - @BeforeEach - public void setup() throws Exception { - String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); - - // Execute create job - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()) - .andExpect( - header() - .string( - HttpHeaders.LOCATION, - lambdaMatcher( - s -> - s.endsWith( - String.format( - "/data-jobs/for-team/%s/jobs/%s", - TEST_TEAM_NAME, TEST_JOB_NAME))))); - } - - @Test - public void testJobCancellation_createDeployExecuteAndCancelJob() throws Exception { - // Take the job zip as byte array - byte[] jobZipBinary = - IOUtils.toByteArray( - getClass().getClassLoader().getResourceAsStream("simple_job_cancel.zip")); - - // Execute job upload with user - MvcResult jobUploadResult = - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .content(jobZipBinary) - .contentType(MediaType.APPLICATION_OCTET_STREAM)) - .andExpect(status().isOk()) - .andReturn(); - - DataJobVersion testDataJobVersion = - new ObjectMapper() - .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); - Assertions.assertNotNull(testDataJobVersion); - - String testJobVersionSha = testDataJobVersion.getVersionSha(); - Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); - - // Setup - String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); - - // Execute build and deploy job - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .content(dataJobDeploymentRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()) - .andReturn(); - - // manually start job execution - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON) - .content( - "{\n" - + " \"args\": {\n" - + " \"key\": \"value\"\n" - + " },\n" - + " \"started_by\": \"schedule/runtime\"\n" - + "}")) - .andExpect(status().is(202)) - .andReturn(); - - // wait for pod to initialize - Thread.sleep(10000); - - // retrieve running job execution id. - var exc = - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - var gson = new Gson(); - ArrayList parsed = - gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); - String executionId = (String) parsed.get(0).get("id"); - - // cancel running execution - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, executionId)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - } - - @Test - public void testJobCancellation_nonExistingJob() throws Exception { - - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, "executionId")) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isNotFound()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java deleted file mode 100644 index 5d403dac32..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java +++ /dev/null @@ -1,460 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.credentials.JobCredentialsService; -import com.vmware.taurus.service.model.DataJob; -import org.apache.commons.lang3.ArrayUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; - -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.NEW_TEST_TEAM_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_CLIENT_ERROR_JOB_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_CLIENT_ERROR_TEAM; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_JOB_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_RETRIED_JOB_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_RETRIED_TEAM; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_TEAM; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_WRONG_NAME; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class DataJobCrudIT extends BaseIT { - - @Autowired private JobsRepository jobsRepository; - - @Test - public void testDataJobCrud() throws Exception { - // Setup - String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); - - // Execute create job (Post Create WebHook will return success for this call) - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()) - .andExpect( - header() - .string( - HttpHeaders.LOCATION, - lambdaMatcher( - s -> - s.endsWith( - String.format( - "/data-jobs/for-team/%s/jobs/%s", - TEST_TEAM_NAME, TEST_JOB_NAME))))); - // Validate - the job is created - Assertions.assertTrue(jobsRepository.existsById(TEST_JOB_NAME)); - - // Execute create job with no user - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isUnauthorized()); - - testDataJobPostCreateWebHooks(); - - // Execute get swagger with no user - mockMvc - .perform( - get("/data-jobs/swagger-ui.html") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - mockMvc - .perform( - get("/data-jobs/webjars/springfox-swagger-ui/swagger-ui.css.map") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - mockMvc - .perform( - get("/data-jobs/webjars/springfox-swagger-ui/swagger-ui-bundle.js.map") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - mockMvc - .perform( - get("/data-jobs/swagger-resources/configuration/ui") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - mockMvc - .perform( - get("/data-jobs/swagger-resources/configuration/security") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - mockMvc - .perform( - get("/data-jobs/swagger-resources") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - mockMvc - .perform( - get("/data-jobs/v2/api-docs") - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - - // Validation - // Validate keytab created - var keytabSecretName = JobCredentialsService.getJobKeytabKubernetesSecretName(TEST_JOB_NAME); - var keytabSecretData = dataJobsKubernetesService.getSecretData(keytabSecretName); - Assertions.assertFalse(keytabSecretData.isEmpty()); - Assertions.assertTrue(keytabSecretData.containsKey("keytab")); - Assertions.assertTrue(ArrayUtils.isNotEmpty(keytabSecretData.get("keytab"))); - - // Validate persisted job - var jobFromDbOptional = jobsRepository.findById(TEST_JOB_NAME); - Assertions.assertTrue(jobFromDbOptional.isPresent()); - var jobFromDb = jobFromDbOptional.get(); - Assertions.assertEquals(TEST_JOB_SCHEDULE, jobFromDb.getJobConfig().getSchedule()); - - // Execute list jobs - // deprecated jobsList in favour of jobsQuery - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param( - "query", - "query($filter: [Predicate], $pageNumber: Int) {" - + " jobs(pageNumber: $pageNumber, filter: $filter) {" - + " content {" - + " jobName" - + " }" - + " }" - + "}") - .param( - "variables", - "{" - + "\"filter\": [" - + " {" - + " \"property\": \"config.team\"," - + " \"pattern\": \"" - + TEST_TEAM_NAME - + "\"" - + " }" - + " ]," - + "\"pageNumber\": 1" - + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(content().string(lambdaMatcher(s -> (s.contains(TEST_JOB_NAME))))); - - // Execute list jobs with no user - // deprecated jobsList in favour of jobsQuery - mockMvc - .perform(get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME))) - .andExpect(status().isUnauthorized()) - .andExpect(content().string(lambdaMatcher(s -> s.isBlank()))); - - // Execute get job with user - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user"))) - .andExpect(status().isOk()) - .andExpect(content().string(lambdaMatcher(s -> s.contains(TEST_JOB_NAME)))) - .andExpect(content().string(lambdaMatcher(s -> s.contains(TEST_JOB_SCHEDULE)))); - - // Execute get job with user and wrong team - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_WRONG_NAME, TEST_JOB_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Execute get job without user - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME))) - .andExpect(status().isUnauthorized()) - .andExpect(content().string(lambdaMatcher(s -> s.isBlank()))); - - // Execute update job team with user - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s/team/%s", - TEST_TEAM_WRONG_NAME, TEST_JOB_NAME, NEW_TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Execute update job team with no user - mockMvc - .perform( - put( - String.format( - "/data-jobs/for-team/%s/jobs/%s/team/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, NEW_TEST_TEAM_NAME))) - .andExpect(status().isUnauthorized()); - - // Execute update job team with same team and user - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s/team/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isOk()); - - // Execute update job team with empty team - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s/team/ ", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Execute update job team with user - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s/team/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isOk()); - - // Execute update job team with non existing job - mockMvc - .perform( - put(String.format("/data-jobs/for-team/%s/jobs/missing-job/team/", TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Execute update job team with empty job name - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/ /team/%s", TEST_TEAM_NAME, NEW_TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Execute update job team with missing job name - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs//team/%s", TEST_TEAM_NAME, NEW_TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Execute get not allowed method to job team endpoint - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/team/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isMethodNotAllowed()); - - // Execute put method with user and with wrong team - // TODO: uncomment and extend tests to cover all put operations not only NotFound - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_WRONG_NAME, TEST_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON) - .content(dataJobRequestBody)) - .andExpect(status().isNotFound()); - - // Execute delete job with no user - mockMvc - .perform( - delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isUnauthorized()); - - // Execute delete job with user and wrong team - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_WRONG_NAME, TEST_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isNotFound()); - - // Execute delete job with user - mockMvc - .perform( - delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - - // Execute update job team after job is deleted from db and is missing - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s/team/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) - .with(user("user"))) - .andExpect(status().isNotFound()); - - // Validate keytab deleted - keytabSecretData = dataJobsKubernetesService.getSecretData(keytabSecretName); - Assertions.assertTrue(keytabSecretData.isEmpty()); - - // Validate job deleted from db - Assertions.assertFalse(jobsRepository.existsById(TEST_JOB_NAME)); - - testDataJobPostDeleteWebHooks(); - } - - private void testDataJobPostCreateWebHooks() throws Exception { - // Post Create WebHook will prevent job creation and the result will be propagated error code - // with no job created - String clientErrorDataJobRequestBody = - getDataJobRequestBody(TEST_CLIENT_ERROR_TEAM, TEST_CLIENT_ERROR_JOB_NAME); - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_CLIENT_ERROR_TEAM)) - .with(user("user")) - .content(clientErrorDataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isBadRequest()); - // Validate - the job is NOT created - Assertions.assertFalse(jobsRepository.existsById(TEST_CLIENT_ERROR_JOB_NAME)); - - // Post Create WebHook will prevent job creation and the result will be error 503 with no job - // created - String internalServerErrorDataJobRequestBody = - getDataJobRequestBody(TEST_INTERNAL_ERROR_TEAM, TEST_INTERNAL_ERROR_JOB_NAME); - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_INTERNAL_ERROR_TEAM)) - .with(user("user")) - .content(internalServerErrorDataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isServiceUnavailable()); - // Validate - the job is NOT created - Assertions.assertFalse(jobsRepository.existsById(TEST_INTERNAL_ERROR_JOB_NAME)); - - // Post Create WebHook will retry 2 times and finally will allow job creation - String retriedErrorDataJobRequestBody = - getDataJobRequestBody( - TEST_INTERNAL_ERROR_RETRIED_TEAM, TEST_INTERNAL_ERROR_RETRIED_JOB_NAME); - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_INTERNAL_ERROR_RETRIED_TEAM)) - .with(user("user")) - .content(retriedErrorDataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()); - // Validate - the job is created - Assertions.assertTrue(jobsRepository.existsById(TEST_INTERNAL_ERROR_RETRIED_JOB_NAME)); - } - - private void testDataJobPostDeleteWebHooks() throws Exception { - // Add test job to the repository - DataJob clientErrorEntity = - getDataJobRepositoryModel(TEST_CLIENT_ERROR_TEAM, TEST_CLIENT_ERROR_JOB_NAME); - jobsRepository.save(clientErrorEntity); - // Post Delete WebHook will prevent job deletion and the result will be propagated error code - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s", - TEST_CLIENT_ERROR_TEAM, TEST_CLIENT_ERROR_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isBadRequest()); - // Validate - the job is NOT deleted - Assertions.assertTrue(jobsRepository.existsById(TEST_CLIENT_ERROR_JOB_NAME)); - // Clean Up - jobsRepository.delete(clientErrorEntity); - - // Add test job to the repository - DataJob internalServerEntity = - getDataJobRepositoryModel(TEST_INTERNAL_ERROR_TEAM, TEST_INTERNAL_ERROR_JOB_NAME); - jobsRepository.save(internalServerEntity); - // Post Delete WebHook will prevent job deletion and the result will be error 503 - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s", - TEST_INTERNAL_ERROR_TEAM, TEST_INTERNAL_ERROR_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isServiceUnavailable()); - // Validate - the job is NOT deleted - Assertions.assertTrue(jobsRepository.existsById(TEST_INTERNAL_ERROR_JOB_NAME)); - // Clean Up - jobsRepository.delete(internalServerEntity); - - // Add test job to the repository - DataJob internalServerRetriedEntity = - getDataJobRepositoryModel( - TEST_INTERNAL_ERROR_RETRIED_TEAM, TEST_INTERNAL_ERROR_RETRIED_JOB_NAME); - jobsRepository.save(internalServerRetriedEntity); - // Post Create WebHook will retry 2 times and finally will allow job creation - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s", - TEST_INTERNAL_ERROR_RETRIED_TEAM, TEST_INTERNAL_ERROR_RETRIED_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - // Validate - the job is deleted - Assertions.assertFalse(jobsRepository.existsById(TEST_INTERNAL_ERROR_RETRIED_JOB_NAME)); - // Clean Up - jobsRepository.delete(internalServerEntity); - } - - @Test - public void testDataJobCreateDeleteIdempotency() throws Exception { - String dataJobTestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); - - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .content(dataJobTestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()); - - mockMvc - .perform( - delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - - mockMvc - .perform( - delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isNotFound()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java deleted file mode 100644 index 48d9a6a4c8..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it; - -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import static org.awaitility.Awaitility.await; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import com.google.gson.internal.LinkedTreeMap; -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.controlplane.model.data.DataJobExecution; -import com.vmware.taurus.controlplane.model.data.DataJobVersion; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.platform.commons.util.StringUtils; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Import; -import org.springframework.http.MediaType; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.web.servlet.MvcResult; -import java.util.concurrent.TimeUnit; -import java.util.List; -import com.fasterxml.jackson.core.type.TypeReference; -import java.util.stream.Collectors; -import org.jetbrains.annotations.NotNull; - -import java.util.ArrayList; -import java.util.UUID; - -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@Slf4j -@Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class}) -@TestPropertySource( - properties = { - "dataJob.readOnlyRootFileSystem=true", - }) -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class DataJobEphemeralStorageIT extends BaseIT { - - private static final String TEST_JOB_NAME = - "ephemeral-storage-test-" + UUID.randomUUID().toString().substring(0, 8); - private static final Object DEPLOYMENT_ID = "testing-ephemeral-storage"; - private final ObjectMapper objectMapper = - new ObjectMapper() - .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; - - @AfterEach - public void cleanUp() throws Exception { - // delete job - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user"))) - .andExpect(status().isOk()); - - // Execute delete deployment - mockMvc - .perform( - delete( - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s", - TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()); - } - - @BeforeEach - public void setup() throws Exception { - String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); - - // Execute create job - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()); - } - - @Test - public void testEphemeralStorageJob() throws Exception { - // Take the job zip as byte array - byte[] jobZipBinary = - IOUtils.toByteArray( - getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); - - // Execute job upload with user - MvcResult jobUploadResult = - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .content(jobZipBinary) - .contentType(MediaType.APPLICATION_OCTET_STREAM)) - .andExpect(status().isOk()) - .andReturn(); - - DataJobVersion testDataJobVersion = - new ObjectMapper() - .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); - Assertions.assertNotNull(testDataJobVersion); - - String testJobVersionSha = testDataJobVersion.getVersionSha(); - Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); - - // Setup - String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); - - // Execute build and deploy job - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) - .with(user("user")) - .content(dataJobDeploymentRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isAccepted()) - .andReturn(); - - String opId = TEST_JOB_NAME + UUID.randomUUID().toString().toLowerCase(); - - // manually start job execution - mockMvc - .perform( - post(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .header(HEADER_X_OP_ID, opId) - .contentType(MediaType.APPLICATION_JSON) - .content( - "{\n" - + " \"args\": {\n" - + " \"key\": \"value\"\n" - + " },\n" - + " \"started_by\": \"schedule/runtime\"\n" - + "}")) - .andExpect(status().is(202)) - .andReturn(); - - // wait for pod to initialize - Thread.sleep(10000); - - // retrieve running job execution id. - var exc = - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - var gson = new Gson(); - ArrayList parsed = - gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); - String executionId = (String) parsed.get(0).get("id"); - - // Check the data job execution status - checkDataJobExecutionStatus( - executionId, - DataJobExecution.StatusEnum.SUCCEEDED, - opId, - TEST_JOB_NAME, - TEST_TEAM_NAME, - "user"); - } - - private void checkDataJobExecutionStatus( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - try { - testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobDeploymentExecutionList( - executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionLogs(executionId, jobName, teamName, username); - } catch (Error e) { - try { - // print logs in case execution has failed - MvcResult dataJobExecutionLogsResult = - getExecuteLogs(executionId, jobName, teamName, username); - log.info( - "Job Execution {} logs:\n{}", - executionId, - dataJobExecutionLogsResult.getResponse().getContentAsString()); - } catch (Error ignore) { - } - throw e; - } - } - - private void testDataJobExecutionRead( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) { - - DataJobExecution[] dataJobExecution = new DataJobExecution[1]; - - await() - .atMost(5, TimeUnit.MINUTES) - .with() - .pollInterval(15, TimeUnit.SECONDS) - .until( - () -> { - String dataJobExecutionReadUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s", - teamName, jobName, executionId); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionReadUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - dataJobExecution[0] = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), - DataJobExecution.class); - if (dataJobExecution[0] == null) { - log.info("No response from server"); - } else { - log.info("Response from server " + dataJobExecution[0].getStatus()); - } - return dataJobExecution[0] != null - && executionStatus.equals(dataJobExecution[0].getStatus()); - }); - - assertDataJobExecutionValid(executionId, executionStatus, opId, dataJobExecution[0], jobName); - } - - private void testDataJobExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobExecutionListUrl = - String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); - Assertions.assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - Assertions.assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); - } - - private void testDataJobDeploymentExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobDeploymentExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - teamName, jobName, "release"); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobDeploymentExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); - Assertions.assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - Assertions.assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); - } - - private void testDataJobExecutionLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); - Assertions.assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); - } - - @NotNull - private MvcResult getExecuteLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - String dataJobExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); - MvcResult dataJobExecutionLogsResult = - mockMvc - .perform(get(dataJobExecutionListUrl).with(user(username))) - .andExpect(status().isOk()) - .andReturn(); - return dataJobExecutionLogsResult; - } - - private void assertDataJobExecutionValid( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - DataJobExecution dataJobExecution, - String jobName) { - - Assertions.assertNotNull(dataJobExecution); - Assertions.assertEquals(executionId, dataJobExecution.getId()); - Assertions.assertEquals(jobName, dataJobExecution.getJobName()); - Assertions.assertEquals(executionStatus, dataJobExecution.getStatus()); - Assertions.assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); - Assertions.assertEquals(opId, dataJobExecution.getOpId()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java deleted file mode 100644 index 4f8f34d5ac..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.MediaType; - -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.NEW_TEST_TEAM_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_1; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_2; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_3; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_4; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_5; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_6; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class DataJobGraphQLIT extends BaseIT { - - private static final String DEFAULT_QUERY_WITH_VARS = - "query($filter: [Predicate], $search: String, $pageNumber: Int, $pageSize: Int) { " - + " jobs(pageNumber: $pageNumber, pageSize: $pageSize, filter: $filter, search: $search)" - + " { content { jobName config { team description " - + " schedule { scheduleCron } } } totalPages totalItems }" - + "}"; - - @Test - public void testGraphQLPagination() throws Exception { - createDummyJobs(); - - // Test listing of team jobs by filtering with team property - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param( - "variables", - "{" - + "\"filter\": [" - + " {" - + " \"property\": \"config.team\"," - + " \"pattern\": \"" - + TEST_TEAM_NAME - + "\"" - + " }" - + " ]," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (s.contains(TEST_JOB_1)) - && (s.contains(TEST_JOB_2)) - && (s.contains(TEST_JOB_6)) - && (!s.contains(TEST_JOB_3)) - && (!s.contains(TEST_JOB_4)) - && (!s.contains(TEST_JOB_5))))); - - // Test listing of team jobs by searching for team name - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param( - "variables", - "{" - + "\"search\": \"" - + NEW_TEST_TEAM_NAME - + "\"," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (!s.contains(TEST_JOB_1)) - && (!s.contains(TEST_JOB_2)) - && (!s.contains(TEST_JOB_6)) - && (s.contains(TEST_JOB_3)) - && (s.contains(TEST_JOB_4)) - && (s.contains(TEST_JOB_5))))); - - // Test showing only first page - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param("variables", "{" + "\"pageNumber\": 1," + "\"pageSize\": 2" + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (s.contains(TEST_JOB_1)) - && (s.contains(TEST_JOB_2)) - && (!s.contains(TEST_JOB_3)) - && (!s.contains(TEST_JOB_4)) - && (!s.contains(TEST_JOB_5)) - && (!s.contains(TEST_JOB_6))))); - - // Test showing only middle page - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param("variables", "{" + "\"pageNumber\": 2," + "\"pageSize\": 2" + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (!s.contains(TEST_JOB_1)) - && (!s.contains(TEST_JOB_2)) - && (s.contains(TEST_JOB_3)) - && (s.contains(TEST_JOB_4)) - && (!s.contains(TEST_JOB_5)) - && (!s.contains(TEST_JOB_6))))); - - // Test showing only last page - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param("variables", "{" + "\"pageNumber\": 3," + "\"pageSize\": 2" + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (!s.contains(TEST_JOB_1)) - && (!s.contains(TEST_JOB_2)) - && (!s.contains(TEST_JOB_3)) - && (!s.contains(TEST_JOB_4)) - && (s.contains(TEST_JOB_5)) - && (s.contains(TEST_JOB_6))))); - - // Test showing only first page sorted DESC by jobName - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param( - "variables", - "{" - + "\"filter\": [" - + " {" - + " \"property\": \"jobName\"," - + " \"sort\": \"DESC\"" - + " }" - + " ]," - + "\"pageNumber\": 1," - + "\"pageSize\": 2" - + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (!s.contains(TEST_JOB_1)) - && (!s.contains(TEST_JOB_2)) - && (!s.contains(TEST_JOB_3)) - && (!s.contains(TEST_JOB_4)) - && (s.contains(TEST_JOB_5)) - && (s.contains(TEST_JOB_6))))); - - // Test showing custom filtered, sorted and paged data jobs by multiple parameters - mockMvc - .perform( - get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .param("query", DEFAULT_QUERY_WITH_VARS) - .param( - "variables", - "{" - + "\"filter\": [" - + " {" - + " \"property\": \"config.team\"," - + " \"pattern\": \"" - + TEST_TEAM_NAME - + "\"" - + " }," - + " {" - + " \"property\": \"jobName\"," - + " \"sort\": \"DESC\"" - + " }" - + " ]," - + "\"pageNumber\": 1," - + "\"pageSize\": 2" - + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect( - content() - .string( - lambdaMatcher( - s -> - (!s.contains(TEST_JOB_1)) - && (s.contains(TEST_JOB_2)) - && (!s.contains(TEST_JOB_3)) - && (!s.contains(TEST_JOB_4)) - && (!s.contains(TEST_JOB_5)) - && (s.contains(TEST_JOB_6))))); - - deleteDummyJobs(); - } - - private void createDummyJobs() throws Exception { - // Setup by creating 3 jobs in 2 separate teams (6 jobs total) - String dataJobTestBodyOne = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_1); - createJob(dataJobTestBodyOne, TEST_TEAM_NAME); - - String dataJobTestBodyTwo = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_2); - createJob(dataJobTestBodyTwo, TEST_TEAM_NAME); - - String dataJobTestBodyThree = getDataJobRequestBody(NEW_TEST_TEAM_NAME, TEST_JOB_3); - createJob(dataJobTestBodyThree, NEW_TEST_TEAM_NAME); - - String dataJobTestBodyFour = getDataJobRequestBody(NEW_TEST_TEAM_NAME, TEST_JOB_4); - createJob(dataJobTestBodyFour, NEW_TEST_TEAM_NAME); - - String dataJobTestBodyFive = getDataJobRequestBody(NEW_TEST_TEAM_NAME, TEST_JOB_5); - createJob(dataJobTestBodyFive, NEW_TEST_TEAM_NAME); - - String dataJobTestBodySix = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_6); - createJob(dataJobTestBodySix, TEST_TEAM_NAME); - } - - private void deleteDummyJobs() throws Exception { - // Clean up - deleteJob(TEST_JOB_1, TEST_TEAM_NAME); - deleteJob(TEST_JOB_2, TEST_TEAM_NAME); - deleteJob(TEST_JOB_3, NEW_TEST_TEAM_NAME); - deleteJob(TEST_JOB_4, NEW_TEST_TEAM_NAME); - deleteJob(TEST_JOB_5, NEW_TEST_TEAM_NAME); - deleteJob(TEST_JOB_6, TEST_TEAM_NAME); - } - - private void createJob(String body, String teamName) throws Exception { - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", teamName)) - .with(user("user")) - .content(body) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()); - } - - private void deleteJob(String jobName, String teamName) throws Exception { - mockMvc - .perform( - delete(String.format("/data-jobs/for-team/%s/jobs/%s", teamName, jobName)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java deleted file mode 100644 index 111ea9f663..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import com.vmware.taurus.properties.service.PropertiesRepository; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; - -import java.util.HashMap; - -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_NAME; -import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class DataJobPropertiesIT extends BaseIT { - - @Autowired private PropertiesRepository propertiesRepository; - - @Test - public void testDataJobProperties() throws Exception { - // Setup - String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); - - // Execute create job (Post Create WebHook will return success for this call) - mockMvc - .perform( - post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) - .with(user("user")) - .content(dataJobRequestBody) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isCreated()) - .andExpect( - header() - .string( - HttpHeaders.LOCATION, - lambdaMatcher( - s -> - s.endsWith( - String.format( - "/data-jobs/for-team/%s/jobs/%s", - TEST_TEAM_NAME, TEST_JOB_NAME))))); - - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/properties", - TEST_TEAM_NAME, TEST_JOB_NAME, "dev")) - .with(user("user"))) - .andExpect(status().isOk()) - .andExpect(content().string("{}")); - - var props = new HashMap(); - props.put("string_key", "string_value"); - props.put("int_key", 123); - props.put("bool_key", true); - mockMvc - .perform( - put(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/properties", - TEST_TEAM_NAME, TEST_JOB_NAME, "dev")) - .with(user("user")) - .content(mapper.writeValueAsString(props)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isNoContent()); - - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/properties", - TEST_TEAM_NAME, TEST_JOB_NAME, "dev")) - .with(user("user"))) - .andExpect(status().isOk()) - .andExpect(content().json(mapper.writeValueAsString(props))); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java deleted file mode 100644 index 11da9c0633..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.actuate.metrics.AutoConfigureMetrics; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MvcResult; - -import com.vmware.taurus.controlplane.model.data.DataJobExecution; -import com.vmware.taurus.controlplane.model.data.DataJobExecutionRequest; -import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; -import com.vmware.taurus.service.JobsRepository; - -@AutoConfigureMetrics -public class DataJobTerminationStatusIT extends BaseDataJobDeploymentIT { - - public static final String INFO_METRICS = "taurus_datajob_info"; - public static final String TERMINATION_STATUS_METRICS = "taurus_datajob_termination_status"; - public static final String HEADER_X_OP_ID = "X-OPID"; - - private static final Logger log = LoggerFactory.getLogger(DataJobTerminationStatusIT.class); - private final ObjectMapper objectMapper = - new ObjectMapper() - .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; - - @Autowired JobsRepository jobsRepository; - - // TODO split this test into job termination status test and data job execution test - /** - * This test aims to validate the data job execution and termination status monitoring logic in - * TPCS. For the purpose it does the following: - * - *
    - *
  • Creates a new data job (simple_job.zip), configured with: - *
      - *
    • schedule (e.g. */2 * * * *) - *
    • notification emails - *
    • enable_execution_notifications set to false - *
    - *
  • Deploys the data job - *
  • Wait for the deployment to complete - *
  • Executes the data job - *
  • Checks data job execution status - *
  • Wait until a single data job execution to complete (using the awaitility for this) - *
  • Checks data job execution status - *
  • Make a rest call to the (/data-jobs/debug/prometheus) - *
  • Parse the result and validate that: - *
      - *
    • there is a taurus_datajob_info metrics for the data job used for testing and that - * it has the appropriate email labels (they should be empty because - * enable_execution_notifications is false) - *
    • there is a taurus_datajob_termination_status metrics for the data job used for - * testing and it has the appropriate value (0.0 in this case; i.e. the job should - * have succeeded) - *
    - *
  • Finally, delete the job deployment and the leftover K8s jobs. - *
- */ - @Test - public void testDataJobTerminationStatus(String jobName, String teamName, String username) - throws Exception { - // Execute data job - String opId = jobName + UUID.randomUUID().toString().toLowerCase(); - DataJobExecutionRequest dataJobExecutionRequest = - new DataJobExecutionRequest().startedBy(username); - - String triggerDataJobExecutionUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - teamName, jobName, "release"); - MvcResult dataJobExecutionResponse = - mockMvc - .perform( - post(triggerDataJobExecutionUrl) - .with(user(username)) - .header(HEADER_X_OP_ID, opId) - .content(mapper.writeValueAsString(dataJobExecutionRequest)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().is(202)) - .andReturn(); - - // Check the data job execution status - String location = dataJobExecutionResponse.getResponse().getHeader("location"); - String executionId = location.substring(location.lastIndexOf("/") + 1); - checkDataJobExecutionStatus( - executionId, DataJobExecution.StatusEnum.SUCCEEDED, opId, jobName, teamName, username); - - // Wait for the job execution to complete, polling every 5 seconds - // See: https://github.com/awaitility/awaitility/wiki/Usage - await() - .atMost(10, TimeUnit.MINUTES) - .with() - .pollInterval(15, TimeUnit.SECONDS) - .until(terminationMetricsAreAvailable()); - - String scrape = scrapeMetrics(); - - // Validate that all metrics for the executed data job are correctly exposed - // First, validate that there is a taurus_datajob_info metrics for the data job - var match = findMetricsWithLabel(scrape, INFO_METRICS, "data_job", jobName); - assertTrue( - match.isPresent(), - String.format("Could not find %s metrics for the data job %s", INFO_METRICS, jobName)); - - // Validate the labels of the taurus_datajob_info metrics - String line = match.get(); - assertLabelEquals(INFO_METRICS, teamName, "team", line); - assertLabelEquals(INFO_METRICS, "", "email_notified_on_success", line); - assertLabelEquals(INFO_METRICS, "", "email_notified_on_platform_error", line); - assertLabelEquals(INFO_METRICS, "", "email_notified_on_user_error", line); - - // Validate that there is a taurus_datajob_info metrics for the data job - match = findMetricsWithLabel(scrape, TERMINATION_STATUS_METRICS, "data_job", jobName); - assertTrue( - match.isPresent(), - String.format( - "Could not find %s metrics for the data job %s", TERMINATION_STATUS_METRICS, jobName)); - - // Validate that the metrics has a value of 0.0 (i.e. Success) - System.out.println(match.get().trim()); - assertTrue( - match.get().trim().endsWith("0.0"), - "The value of the taurus_datajob_termination_status metrics does not match. It was actually" - + " " - + match.get()); - - // Check the data job execution status - checkDataJobExecutionStatus( - executionId, DataJobExecution.StatusEnum.SUCCEEDED, opId, jobName, teamName, username); - } - - private String scrapeMetrics() throws Exception { - return mockMvc - .perform(get("/data-jobs/debug/prometheus")) - .andExpect(status().isOk()) - .andReturn() - .getResponse() - .getContentAsString(); - } - - private Callable terminationMetricsAreAvailable() { - return () -> scrapeMetrics().contains(TERMINATION_STATUS_METRICS); - } - - private static Optional findMetricsWithLabel( - CharSequence input, String metrics, String label, String labelValue) { - Pattern pattern = - Pattern.compile( - String.format("%s\\{.*%s=\"%s\"[^\\}]*\\} (.*)", metrics, label, labelValue), - Pattern.CASE_INSENSITIVE); - Matcher matcher = pattern.matcher(input); - if (!matcher.find()) { - return Optional.empty(); - } - return Optional.of(matcher.group()); - } - - private static void assertLabelEquals( - String metrics, String expectedValue, String label, String line) { - // Label value is captured in the first regex group - Matcher matcher = - Pattern.compile(String.format(".*%s=\"([^\"]*)\".*", label), Pattern.CASE_INSENSITIVE) - .matcher(line); - assertTrue( - matcher.find(), - String.format("The metrics %s does not have a matching label %s", metrics, label)); - String actualValue = matcher.group(1); - assertEquals( - expectedValue, - actualValue, - String.format( - "The metrics %s does not have correct value for label %s. Expected: %s, Actual: %s", - metrics, label, expectedValue, actualValue)); - } - - private void checkDataJobExecutionStatus( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - try { - testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobDeploymentExecutionList( - executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionLogs(executionId, jobName, teamName, username); - } catch (Error e) { - try { - // print logs in case execution has failed - MvcResult dataJobExecutionLogsResult = - getExecuteLogs(executionId, jobName, teamName, username); - log.info( - "Job Execution {} logs:\n{}", - executionId, - dataJobExecutionLogsResult.getResponse().getContentAsString()); - } catch (Error ignore) { - } - throw e; - } - } - - private void testDataJobExecutionRead( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) { - - DataJobExecution[] dataJobExecution = new DataJobExecution[1]; - - await() - .atMost(5, TimeUnit.MINUTES) - .with() - .pollInterval(15, TimeUnit.SECONDS) - .until( - () -> { - String dataJobExecutionReadUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s", - teamName, jobName, executionId); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionReadUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - dataJobExecution[0] = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), - DataJobExecution.class); - if (dataJobExecution[0] == null) { - log.info("No response from server"); - } else { - log.info("Response from server " + dataJobExecution[0].getStatus()); - } - return dataJobExecution[0] != null - && executionStatus.equals(dataJobExecution[0].getStatus()); - }); - - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecution[0], jobName, username); - } - - private void testDataJobExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobExecutionListUrl = - String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); - assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName, username); - } - - private void testDataJobDeploymentExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobDeploymentExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - teamName, jobName, "release"); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobDeploymentExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); - assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName, username); - } - - private void testDataJobExecutionLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); - assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); - } - - @NotNull - private MvcResult getExecuteLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - String dataJobExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); - MvcResult dataJobExecutionLogsResult = - mockMvc - .perform(get(dataJobExecutionListUrl).with(user(username))) - .andExpect(status().isOk()) - .andReturn(); - return dataJobExecutionLogsResult; - } - - private void assertDataJobExecutionValid( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - DataJobExecution dataJobExecution, - String jobName, - String username) { - - assertNotNull(dataJobExecution); - assertEquals(executionId, dataJobExecution.getId()); - assertEquals(jobName, dataJobExecution.getJobName()); - assertEquals(executionStatus, dataJobExecution.getStatus()); - assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); - assertEquals(username + "/" + "user", dataJobExecution.getStartedBy()); - assertEquals(opId, dataJobExecution.getOpId()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java deleted file mode 100644 index 5527cad0cd..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it.common; - -import static org.awaitility.Awaitility.await; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; - -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.core.task.SyncTaskExecutor; -import org.springframework.core.task.TaskExecutor; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MvcResult; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.controlplane.model.data.DataJobExecutionRequest; - -/** - * It combines all necessary annotations and constants for tests that need an already deployed data - * job. - * - *

The test just needs to extend this class, and it will have access to the already deployed data - * job. - */ -@Import({BaseDataJobDeploymentIT.TaskExecutorConfig.class}) -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -@ExtendWith(DataJobDeploymentExtension.class) -public abstract class BaseDataJobDeploymentIT extends BaseIT { - - @TestConfiguration - public static class TaskExecutorConfig { - @Bean - @Primary - public TaskExecutor taskExecutor() { - // Deployment methods are non-blocking (Async) which makes them harder to test. - // Making them sync for the purposes of this test. - return new SyncTaskExecutor(); - } - } - - protected MvcResult executeDataJob( - String jobName, String teamName, String username, String opId) { - // Execute data job - if (opId == null) { - opId = jobName + UUID.randomUUID().toString().toLowerCase(); - } - - DataJobExecutionRequest dataJobExecutionRequest = - new DataJobExecutionRequest().startedBy(username); - - String triggerDataJobExecutionUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - teamName, jobName, "release"); - - // Wait for the job execution to complete, polling every 15 seconds - // See: https://github.com/awaitility/awaitility/wiki/Usage - String finalOpId = opId; - return await() - .atMost(5, TimeUnit.MINUTES) - .with() - .pollInterval(15, TimeUnit.SECONDS) - .until( - () -> - mockMvc - .perform( - post(triggerDataJobExecutionUrl) - .with(user(username)) - .header(HEADER_X_OP_ID, finalOpId) - .content(mapper.writeValueAsString(dataJobExecutionRequest)) - .contentType(MediaType.APPLICATION_JSON)) - .andReturn(), - mvcResult -> mvcResult.getResponse().getStatus() == 202); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java deleted file mode 100644 index 9d750cccf8..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it.common; - -import java.time.OffsetDateTime; - -import com.vmware.taurus.service.JobExecutionRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.DataJobExecution; -import com.vmware.taurus.service.model.ExecutionStatus; -import com.vmware.taurus.service.model.ExecutionType; - -public class JobExecutionUtil { - - public static DataJobExecution createDataJobExecution( - JobExecutionRepository jobExecutionRepository, - String executionId, - DataJob dataJob, - OffsetDateTime startTime, - OffsetDateTime endTime, - ExecutionStatus executionStatus) { - - var jobExecution = - DataJobExecution.builder() - .id(executionId) - .dataJob(dataJob) - .startTime(startTime) - .endTime(endTime) - .type(ExecutionType.MANUAL) - .status(executionStatus) - .resourcesCpuRequest(1F) - .resourcesCpuLimit(2F) - .resourcesMemoryRequest(500) - .resourcesMemoryLimit(1000) - .message("message") - .lastDeployedBy("test_user") - .lastDeployedDate(OffsetDateTime.now()) - .jobVersion("test_version") - .jobSchedule("*/5 * * * *") - .opId("test_op_id") - .vdkVersion("test_vdk_version") - .build(); - - return jobExecutionRepository.save(jobExecution); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java deleted file mode 100644 index 1697921b1d..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it.common; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.springframework.security.kerberos.test.KerberosSecurityTestcase; - -public class KerberosSecurityTestcaseJunit5 extends KerberosSecurityTestcase { - // TODO we should think about moving away from MiniKDC as it's not maintained. - @BeforeEach - @Override - public void startMiniKdc() throws Exception { - super.startMiniKdc(); - } - - @AfterEach - @Override - public void stopMiniKdc() { - super.stopMiniKdc(); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java deleted file mode 100644 index 4850d95cea..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.datajobs.it.common; - -import com.vmware.taurus.service.credentials.KerberosCredentialsRepository; -import lombok.Setter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.security.kerberos.test.MiniKdc; - -import java.io.File; -import java.util.Optional; - -/** - * This class can be used to replace {@link KerberosCredentialsRepository} for testing purposes. It - * uses {@link MiniKdc} as a kerberos server. - * - *

TODO: Figure out how to connect to a testing KDC server via kadmin * For some reason I - * (tsvetkovt@vmware.com) was not able to connect to MiniKdc or to a * dockerized KDC with kadmin. - * TODO we should think about moving away from MiniKDC as it's not maintained. - */ -@Setter -public class MiniKdcCredentialsRepository extends KerberosCredentialsRepository { - private static final Logger log = LoggerFactory.getLogger(MiniKdcCredentialsRepository.class); - - private MiniKdc miniKdc; - - public MiniKdcCredentialsRepository() { - super("test", "test"); - } - - @Override - public void createPrincipal(String principal, Optional keytabLocation) { - try { - miniKdc.createPrincipal(keytabLocation.get(), principal); - } catch (Exception e) { - log.error("Failed to create principal", e); - } - } - - @Override - public boolean principalExists(String principal) { - return false; - } - - @Override - public void deletePrincipal(String principal) { - // Principals are deleted when MiniKdc is shut down. - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java deleted file mode 100644 index a3c1459272..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.ExecutionStatus; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; - -import java.time.OffsetDateTime; -import java.time.ZoneOffset; - -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -public class GraphQLDataJobsFieldsIT extends BaseDataJobDeploymentIT { - - private static final String DEFAULT_QUERY_WITH_DEPLOYMENTS = - "query($filter: [Predicate], $executionOrder: DataJobExecutionOrder, $search: String," - + " $pageNumber: Int, $pageSize: Int) { jobs(pageNumber: $pageNumber, pageSize:" - + " $pageSize, filter: $filter, search: $search) { content { jobName " - + " deployments { id enabled lastExecutionStatus " - + " lastExecutionTime lastExecutionDuration executions(pageNumber: 1," - + " pageSize: 5, order: $executionOrder) { id status } } " - + " config { team description schedule { scheduleCron " - + " nextRunEpochSeconds } } } totalPages totalItems }}"; - - @Autowired private JobsRepository jobsRepository; - - @Test - void testFields(String jobName, String teamName, String username) throws Exception { - var dataJob = jobsRepository.findById(jobName).get(); - dataJob.setLastExecutionStatus(ExecutionStatus.SUCCEEDED); - dataJob.setLastExecutionEndTime(OffsetDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)); - dataJob.setLastExecutionDuration(1000); - dataJob = jobsRepository.save(dataJob); - - // Test requesting of fields that are computed - mockMvc - .perform( - get(JOBS_URI) - .with(user(username)) - .param("query", DEFAULT_QUERY_WITH_DEPLOYMENTS) - .param( - "variables", - "{" - + "\"search\": \"" - + jobName - + "\"," - + "\"pageNumber\": 1," - + "\"pageSize\": 10," - + "\"executionOrder\": {" - + " \"direction\": \"DESC\"," - + " \"property\": \"status\"" - + " }" - + "}") - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.data.content[0].config.team", is(teamName))) - .andExpect( - jsonPath( - "$.data.content[0].config.schedule.scheduleCron", - is(dataJob.getJobConfig().getSchedule()))) - .andExpect( - jsonPath("$.data.content[0].config.schedule.nextRunEpochSeconds", greaterThan(1))) - .andExpect( - jsonPath( - "$.data.content[0].deployments[0].lastExecutionStatus", - is(dataJob.getLastExecutionStatus().name()))) - .andExpect( - jsonPath( - "$.data.content[0].deployments[0].lastExecutionTime", - isDate(dataJob.getLastExecutionEndTime()))) - .andExpect( - jsonPath( - "$.data.content[0].deployments[0].lastExecutionDuration", - is(dataJob.getLastExecutionDuration()))) - .andReturn() - .getResponse() - .getContentAsString(); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java deleted file mode 100644 index f71204770d..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.controlplane.model.data.DataJobContacts; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.JobConfig; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - -import java.util.List; - -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class GraphQLDataJobsSortContactsIT extends BaseIT { - - @Autowired private JobsRepository jobsRepository; - - @AfterEach - public void cleanup() { - jobsRepository.deleteAll(); - } - - private String getQuery(String sortOrder) { - return "{\n" - + " jobs(\n" - + " pageNumber: 1\n" - + " pageSize: 5\n" - + " filter: [{ property: \"config.contacts.present\", sort:" - + sortOrder - + "}]\n" - + " ) {\n" - + " content {\n" - + " jobName\n" - + " config {\n" - + " contacts{" - + " notifiedOnJobFailureUserError " - + " notifiedOnJobSuccess" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}"; - } - - @Test - public void testEmptyCall_shouldBeEmpty() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("DESC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content").isEmpty()); - } - - @Test - public void testSingleJob_shouldReturnOne() throws Exception { - var contacts = new DataJobContacts(); - contacts.setNotifiedOnJobFailureUserError(List.of("test-notified")); - var job = createDummyJob(contacts, "test-job"); - jobsRepository.save(job); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("DESC")) - .with(user(TEST_USERNAME))) - .andExpect( - jsonPath("$.data.content[0].config.contacts.notifiedOnJobFailureUserError") - .value("test-notified")); - } - - @Test - public void testSortingTwoJobs_shouldReturnDescOrder() throws Exception { - var contacts = new DataJobContacts(); - contacts.setNotifiedOnJobFailureUserError(List.of("test-notified")); - var job = createDummyJob(contacts, "test-job"); - - var contacts2 = new DataJobContacts(); - var job2 = createDummyJob(contacts2, "test-job2"); - - jobsRepository.save(job); - jobsRepository.save(job2); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("DESC")) - .with(user(TEST_USERNAME))) - .andExpect( - jsonPath("$.data.content[0].config.contacts.notifiedOnJobFailureUserError") - .value("test-notified")) - .andExpect( - jsonPath("$.data.content[1].config.contacts.notifiedOnJobFailureUserError").isEmpty()); - } - - @Test - public void testSortingTwoJobs_shouldReturnAscOrder() throws Exception { - var contacts = new DataJobContacts(); - contacts.setNotifiedOnJobSuccess(List.of("test-notified")); - var job = createDummyJob(contacts, "test-job"); - - var contacts2 = new DataJobContacts(); - var job2 = createDummyJob(contacts2, "test-job2"); - - jobsRepository.save(job); - jobsRepository.save(job2); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("ASC")) - .with(user(TEST_USERNAME))) - .andExpect( - jsonPath("$.data.content[1].config.contacts.notifiedOnJobSuccess") - .value("test-notified")) - .andExpect(jsonPath("$.data.content[0].config.contacts.notifiedOnJobSuccess").isEmpty()); - } - - @Test - public void testSortingTwoJobsNoContacts_shouldReturnTwo() throws Exception { - var contacts = new DataJobContacts(); - var job = createDummyJob(contacts, "test-job"); - - var contacts2 = new DataJobContacts(); - var job2 = createDummyJob(contacts2, "test-job2"); - - jobsRepository.save(job); - jobsRepository.save(job2); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("ASC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content.length()").value(2)) - .andExpect( - jsonPath("$.data.content[*].jobName", Matchers.contains("test-job", "test-job2"))); - } - - @Test - public void testSortingTwoJobsWithContacts_shouldReturnTwo() throws Exception { - var contacts = new DataJobContacts(); - contacts.setNotifiedOnJobDeploy(List.of("test")); - var job = createDummyJob(contacts, "test-job"); - - var contacts2 = new DataJobContacts(); - contacts2.setNotifiedOnJobFailurePlatformError(List.of("test")); - var job2 = createDummyJob(contacts2, "test-job2"); - - jobsRepository.save(job); - jobsRepository.save(job2); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("ASC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content.length()").value(2)) - .andExpect( - jsonPath("$.data.content[*].jobName", Matchers.contains("test-job", "test-job2"))); - } - - private DataJob createDummyJob(DataJobContacts contacts, String id) { - DataJob job = new DataJob(); - job.setName(id); - JobConfig config = new JobConfig(); - config.setSchedule("test-schedule"); - config.setNotifiedOnJobFailureUserError(contacts.getNotifiedOnJobFailureUserError()); - config.setNotifiedOnJobDeploy(contacts.getNotifiedOnJobDeploy()); - config.setNotifiedOnJobSuccess(contacts.getNotifiedOnJobSuccess()); - config.setNotifiedOnJobFailurePlatformError(contacts.getNotifiedOnJobFailurePlatformError()); - job.setJobConfig(config); - return job; - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java deleted file mode 100644 index 1cde318a6c..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import com.vmware.taurus.datajobs.it.common.JobExecutionUtil; -import com.vmware.taurus.service.JobExecutionRepository; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.DataJobExecution; -import com.vmware.taurus.service.model.ExecutionStatus; -import com.vmware.taurus.service.model.JobConfig; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - -import java.time.OffsetDateTime; -import java.util.List; - -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class GraphQLExecutionsIT extends BaseIT { - - @Autowired JobExecutionRepository jobExecutionRepository; - - @Autowired JobsRepository jobsRepository; - - private static final String TEST_JOB_NAME_1 = "TEST-JOB-NAME-1"; - private static final String TEST_JOB_NAME_2 = "TEST-JOB-NAME-2"; - private static final String TEST_JOB_NAME_3 = "TEST-JOB-NAME-3"; - - private DataJob dataJob1; - private DataJob dataJob2; - private DataJob dataJob3; - - private DataJobExecution dataJobExecution1; - private DataJobExecution dataJobExecution2; - private DataJobExecution dataJobExecution3; - - @BeforeEach - public void setup() { - cleanup(); - - var config1 = new JobConfig(); - config1.setTeam("test-team1"); - - var config2 = new JobConfig(); - config2.setTeam("test-team2"); - - var config3 = new JobConfig(); - config3.setTeam("test-team3"); - - this.dataJob1 = jobsRepository.save(new DataJob(TEST_JOB_NAME_1, config1)); - this.dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, config2)); - this.dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, config3)); - - OffsetDateTime now = OffsetDateTime.now(); - this.dataJobExecution1 = - JobExecutionUtil.createDataJobExecution( - jobExecutionRepository, "testId1", dataJob1, now, now, ExecutionStatus.SUCCEEDED); - this.dataJobExecution2 = - JobExecutionUtil.createDataJobExecution( - jobExecutionRepository, - "testId2", - dataJob2, - now.minusSeconds(1), - now.minusSeconds(1), - ExecutionStatus.USER_ERROR); - this.dataJobExecution3 = - JobExecutionUtil.createDataJobExecution( - jobExecutionRepository, - "testId3", - dataJob3, - now.minusSeconds(10), - now.minusSeconds(10), - ExecutionStatus.SUBMITTED); - } - - private static String getQuery() { - return "query($filter: DataJobExecutionFilter, $order: DataJobExecutionOrder, $pageNumber: Int," - + " $pageSize: Int) { executions(pageNumber: $pageNumber, pageSize: $pageSize," - + " filter: $filter, order: $order) { content { id jobName " - + " startTime endTime status deployment { id enabled " - + " jobVersion deployedDate deployedBy resources { " - + " cpuLimit cpuRequest memoryLimit memoryRequest " - + " } schedule { scheduleCron } vdkVersion } " - + " } totalPages totalItems }}"; - } - - private void cleanup() { - jobsRepository - .findAllById(List.of(TEST_JOB_NAME_1, TEST_JOB_NAME_2, TEST_JOB_NAME_3)) - .forEach(dataJob -> jobsRepository.delete(dataJob)); - } - - @Test - public void testExecutions_filterByStartTimeGte_shouldReturnAllProperties() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"startTimeGte\": \"" - + dataJobExecution2.getStartTime() - + "\"" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user(TEST_USERNAME))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.contains(dataJobExecution1.getId(), dataJobExecution2.getId()))) - .andExpect( - jsonPath( - "$.data.content[*].jobName", - Matchers.contains(dataJob1.getName(), dataJob2.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains( - dataJobExecution1.getStatus().toString(), - dataJobExecution2.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.jobVersion", - Matchers.contains( - dataJobExecution1.getJobVersion(), dataJobExecution2.getJobVersion()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.deployedDate", - Matchers.contains( - dataJobExecution1.getLastDeployedDate().toString(), - dataJobExecution2.getLastDeployedDate().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.deployedBy", - Matchers.contains( - dataJobExecution1.getLastDeployedBy(), dataJobExecution2.getLastDeployedBy()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.vdkVersion", - Matchers.contains( - dataJobExecution1.getVdkVersion(), dataJobExecution2.getVdkVersion()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.resources.cpuRequest", - Matchers.contains( - dataJobExecution1.getResourcesCpuRequest().doubleValue(), - dataJobExecution2.getResourcesCpuRequest().doubleValue()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.resources.cpuLimit", - Matchers.contains( - dataJobExecution1.getResourcesCpuLimit().doubleValue(), - dataJobExecution2.getResourcesCpuLimit().doubleValue()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.resources.memoryRequest", - Matchers.contains( - dataJobExecution1.getResourcesMemoryRequest(), - dataJobExecution2.getResourcesMemoryRequest()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.resources.memoryLimit", - Matchers.contains( - dataJobExecution1.getResourcesMemoryLimit(), - dataJobExecution2.getResourcesMemoryLimit()))) - .andExpect( - jsonPath( - "$.data.content[*].deployment.schedule.scheduleCron", - Matchers.contains( - dataJobExecution1.getJobSchedule(), dataJobExecution2.getJobSchedule()))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution3.getId())))); - } - - @Test - public void testExecutions_filterByStartTimeLte() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"startTimeLte\": \"" - + dataJobExecution2.getStartTime() - + "\"" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user(TEST_USERNAME))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.contains(dataJobExecution2.getId(), dataJobExecution3.getId()))) - .andExpect( - jsonPath( - "$.data.content[*].jobName", - Matchers.contains(dataJob2.getName(), dataJob3.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains( - dataJobExecution2.getStatus().toString(), - dataJobExecution3.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution1.getId())))); - } - - @Test - public void testExecutions_filterByEndTimeGte() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"endTimeGte\": \"" - + dataJobExecution2.getEndTime() - + "\"" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user("user"))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.contains(dataJobExecution1.getId(), dataJobExecution2.getId()))) - .andExpect( - jsonPath( - "$.data.content[*].jobName", - Matchers.contains(dataJob1.getName(), dataJob2.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains( - dataJobExecution1.getStatus().toString(), - dataJobExecution2.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution3.getId())))); - } - - @Test - public void testExecutions_filterByEndTimeLte() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"endTimeLte\": \"" - + dataJobExecution2.getEndTime() - + "\"" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user("user"))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.contains(dataJobExecution2.getId(), dataJobExecution3.getId()))) - .andExpect( - jsonPath( - "$.data.content[*].jobName", - Matchers.contains(dataJob2.getName(), dataJob3.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains( - dataJobExecution2.getStatus().toString(), - dataJobExecution3.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution1.getId())))); - } - - @Test - public void testExecutions_filterByStatusIn() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"statusIn\": [\"" - + dataJobExecution1.getStatus().toString() - + "\", \"" - + dataJobExecution2.getStatus().toString() - + "\"]" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user("user"))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.contains(dataJobExecution1.getId(), dataJobExecution2.getId()))) - .andExpect( - jsonPath( - "$.data.content[*].jobName", - Matchers.contains(dataJob1.getName(), dataJob2.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains( - dataJobExecution1.getStatus().toString(), - dataJobExecution2.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution3.getId())))); - } - - @Test - public void testExecutions_filterByJobNameIn() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"jobNameIn\": [\"" - + dataJobExecution1.getDataJob().getName() - + "\"]" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user("user"))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[*].id", Matchers.contains(dataJobExecution1.getId()))) - .andExpect(jsonPath("$.data.content[*].jobName", Matchers.contains(dataJob1.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains(dataJobExecution1.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution3.getId())))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution2.getId())))); - } - - @Test - public void testExecutions_filterByTeamNameIn() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"teamNameIn\": [\"" - + dataJobExecution1.getDataJob().getJobConfig().getTeam() - + "\"]" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user("user"))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[*].id", Matchers.contains(dataJobExecution1.getId()))) - .andExpect(jsonPath("$.data.content[*].jobName", Matchers.contains(dataJob1.getName()))) - .andExpect( - jsonPath( - "$.data.content[*].status", - Matchers.contains(dataJobExecution1.getStatus().toString()))) - .andExpect( - jsonPath( - "$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution3.getId())))) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.not(Matchers.contains(dataJobExecution2.getId())))); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java deleted file mode 100644 index b073ff07bc..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.text.MessageFormat; -import java.time.OffsetDateTime; -import java.util.List; - -import org.hamcrest.Matchers; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import com.vmware.taurus.datajobs.it.common.JobExecutionUtil; -import com.vmware.taurus.service.JobExecutionRepository; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.DataJobExecution; -import com.vmware.taurus.service.model.ExecutionStatus; -import com.vmware.taurus.service.model.JobConfig; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class GraphQLExecutionsLogsUrlIT extends BaseIT { - - @Autowired JobExecutionRepository jobExecutionRepository; - - @Autowired JobsRepository jobsRepository; - - private static final String LOGS_URL_TEMPLATE = - "https://log-insight-base-url/li/query/stream?query=%C2%A7%C2%A7%C2%A7AND%C" - + "2%A7%C2%A7%C2%A7%C2%A7{0}%C2%A7{1}%C2%A7true%C2%A7COUNT%C2%A7*%C2%A7" - + "timestamp%C2%A7pageSortPreference:%7B%22sortBy%22%3A%22-{2}-{3}-" - + "ingest_timestamp%22%2C%22sortOrder%22%3A%22DESC%22%7D%C2%A7" - + "alertDefList:%5B%5D%C2%A7partitions:%C2%A7%C2%A7text:CONTAINS:{4}*"; - - private static final String TEST_JOB_NAME_1 = "TEST-JOB-NAME-1"; - private static final String TEST_JOB_NAME_2 = "TEST-JOB-NAME-2"; - private static final String TEST_JOB_NAME_3 = "TEST-JOB-NAME-3"; - - private static final long TEST_START_TIME_OFFSET_SECONDS = 1000; - private static final long TEST_END_TIME_OFFSET_SECONDS = -1000; - - private DataJobExecution dataJobExecution1; - private DataJobExecution dataJobExecution2; - private DataJobExecution dataJobExecution3; - - @DynamicPropertySource - static void dynamicProperties(DynamicPropertyRegistry registry) { - String logsUrlTemplate = - MessageFormat.format( - LOGS_URL_TEMPLATE, - "{{start_time}}", - "{{end_time}}", - "{{job_name}}", - "{{op_id}}", - "{{execution_id}}"); - registry.add("datajobs.executions.logsUrl.template", () -> logsUrlTemplate); - registry.add("datajobs.executions.logsUrl.dateFormat", () -> "unix"); - registry.add( - "datajobs.executions.logsUrl.startTimeOffsetSeconds", () -> TEST_START_TIME_OFFSET_SECONDS); - registry.add( - "datajobs.executions.logsUrl.endTimeOffsetSeconds", () -> TEST_END_TIME_OFFSET_SECONDS); - } - - @BeforeEach - public void setup() { - cleanup(); - - DataJob dataJob1 = jobsRepository.save(new DataJob(TEST_JOB_NAME_1, new JobConfig())); - DataJob dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, new JobConfig())); - DataJob dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, new JobConfig())); - - OffsetDateTime now = OffsetDateTime.now(); - this.dataJobExecution1 = - JobExecutionUtil.createDataJobExecution( - jobExecutionRepository, "testId1", dataJob1, now, now, ExecutionStatus.SUCCEEDED); - this.dataJobExecution2 = - JobExecutionUtil.createDataJobExecution( - jobExecutionRepository, - "testId2", - dataJob2, - now.minusSeconds(1), - now.minusSeconds(1), - ExecutionStatus.RUNNING); - this.dataJobExecution3 = - JobExecutionUtil.createDataJobExecution( - jobExecutionRepository, - "testId3", - dataJob3, - now.minusSeconds(10), - now.minusSeconds(10), - ExecutionStatus.SUBMITTED); - } - - private static String getQuery() { - return "query($filter: DataJobExecutionFilter, $order: DataJobExecutionOrder, $pageNumber: Int," - + " $pageSize: Int) { executions(pageNumber: $pageNumber, pageSize: $pageSize," - + " filter: $filter, order: $order) { content { id logsUrl } " - + " totalPages totalItems }}"; - } - - private void cleanup() { - jobsRepository - .findAllById(List.of(TEST_JOB_NAME_1, TEST_JOB_NAME_2, TEST_JOB_NAME_3)) - .forEach(dataJob -> jobsRepository.delete(dataJob)); - } - - @Test - public void testExecutions_filterByStartTimeGte() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery()) - .param( - "variables", - "{" - + "\"filter\": {" - + " \"startTimeGte\": \"" - + dataJobExecution3.getStartTime() - + "\"" - + " }," - + "\"pageNumber\": 1," - + "\"pageSize\": 10" - + "}") - .with(user(TEST_USERNAME))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect( - jsonPath( - "$.data.content[*].id", - Matchers.contains( - dataJobExecution1.getId(), - dataJobExecution2.getId(), - dataJobExecution3.getId()))) - .andExpect( - jsonPath( - "$.data.content[*].logsUrl", - Matchers.contains( - buildLogsUrl(dataJobExecution1), - buildLogsUrl(dataJobExecution2), - buildLogsUrl(dataJobExecution3)))); - } - - private static String buildLogsUrl(DataJobExecution dataJobExecution) { - return MessageFormat.format( - LOGS_URL_TEMPLATE, - String.valueOf( - dataJobExecution - .getStartTime() - .toInstant() - .plusSeconds(TEST_START_TIME_OFFSET_SECONDS) - .toEpochMilli()), - String.valueOf( - dataJobExecution - .getEndTime() - .toInstant() - .plusSeconds(TEST_END_TIME_OFFSET_SECONDS) - .toEpochMilli()), - dataJobExecution.getDataJob().getName(), - dataJobExecution.getOpId(), - dataJobExecution.getId()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java deleted file mode 100644 index 7757d33a61..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.datajobs.it.common.BaseIT; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.DeploymentStatus; -import com.vmware.taurus.service.model.JobConfig; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - -import java.time.OffsetDateTime; -import java.time.ZoneId; - -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = ControlplaneApplication.class) -public class GraphQLExecutionsNextRunIT extends BaseIT { - - @Autowired JobsRepository jobsRepository; - - @BeforeEach - public void cleanup() { - jobsRepository.deleteAll(); - } - - private String getQuery(String sortOrder) { - return "{\n" - + " jobs(pageNumber: 1, pageSize: 100, filter: [{property:" - + " \"config.schedule.nextRunEpochSeconds\", sort:" - + sortOrder - + "}]) {\n" - + " content {\n" - + " jobName \n" - + " config {\n" - + " schedule {\n" - + " scheduleCron\n" - + " nextRunEpochSeconds\n" - + " }" - + " }" - + " }\n" - + " }\n" - + "}"; - } - - private String getQueryWithFilter(String filter) { - return "{\n" - + " jobs(pageNumber: 1, pageSize: 100, filter: [{property:" - + " \"config.schedule.nextRunEpochSeconds\", pattern:" - + filter - + "}]) {\n" - + " content {\n" - + " jobName \n" - + " config {\n" - + " schedule {\n" - + " scheduleCron\n" - + " nextRunEpochSeconds\n" - + " }" - + " }" - + " }\n" - + " }\n" - + "}"; - } - - @Test - public void testNextRunCall_noJobs() throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("DESC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content").isEmpty()); - } - - @Test - public void testNextRunCall_oneJob() throws Exception { - var now = OffsetDateTime.now(); - addJob("jobA", toCron(now)); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("DESC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content[0].jobName").value("jobA")) - .andExpect(jsonPath("$.data.content[0].config.schedule.scheduleCron").value(toCron(now))); - } - - @Test - public void testNextRunCall_twoJobs_DESC() throws Exception { - var now = OffsetDateTime.now(ZoneId.of("UTC")).plusMinutes(2); - var later = now.plusDays(1); - - addJob("jobA", toCron(now)); - addJob("jobB", toCron(later)); - // check correct sort is applied - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("DESC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content[0].jobName").value("jobB")) - .andExpect(jsonPath("$.data.content[1].jobName").value("jobA")) - .andExpect(jsonPath("$.data.content[0].config.schedule.scheduleCron").value(toCron(later))) - .andExpect(jsonPath("$.data.content[1].config.schedule.scheduleCron").value(toCron(now))); - } - - @Test - public void testNextRunCall_twoJobs_ASC() throws Exception { - var now = OffsetDateTime.now(ZoneId.of("UTC")).plusMinutes(2); - var later = now.plusMinutes(60); - - addJob("jobA", toCron(now)); - addJob("jobB", toCron(later)); - // check correct sort is applied - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery("ASC")) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content[0].jobName").value("jobA")) - .andExpect(jsonPath("$.data.content[1].jobName").value("jobB")) - .andExpect(jsonPath("$.data.content[0].config.schedule.scheduleCron").value(toCron(now))) - .andExpect(jsonPath("$.data.content[1].config.schedule.scheduleCron").value(toCron(later))); - } - - @Test - public void testNextRunCall_filter_noJobs() throws Exception { - var now = OffsetDateTime.now(); - var before = now.minusDays(2); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQueryWithFilter(getFilterPattern(before, now))) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content").isEmpty()); - } - - @Test - public void testNextRunCall_filter_twoJobs() throws Exception { - var now = OffsetDateTime.now(ZoneId.of("UTC")); - var after = now.plusMinutes(120); - - addJob("jobA", toCron(now.minusMinutes(240))); - addJob("jobB", toCron(now.plusMinutes(30))); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQueryWithFilter(getFilterPattern(now, after))) - .with(user(TEST_USERNAME))) - .andExpect(jsonPath("$.data.content[0].jobName").value("jobB")) - .andExpect(jsonPath("$.data.content[*].jobName", Matchers.not(Matchers.contains("jobA")))); - } - - private void addJob(String jobName, String jobSchedule) { - JobConfig config = new JobConfig(); - config.setSchedule(jobSchedule); - var job = new DataJob(jobName, config); - job.setLatestJobDeploymentStatus(DeploymentStatus.SUCCESS); - jobsRepository.save(job); - } - - private String toCron(OffsetDateTime dateTime) { - return String.format( - "%d %d %d %d %d", - dateTime.getMinute(), - dateTime.getHour(), - dateTime.getDayOfMonth(), - dateTime.getMonth().getValue(), - dateTime.getDayOfWeek().getValue()); - } - - private String getFilterPattern(OffsetDateTime start, OffsetDateTime finish) { - return String.format("\"%d-%d\"", start.toEpochSecond(), finish.toEpochSecond()); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java deleted file mode 100644 index 7f197bae63..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.time.OffsetDateTime; -import java.util.UUID; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - -import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; -import com.vmware.taurus.service.JobExecutionRepository; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.DataJobExecution; -import com.vmware.taurus.service.model.ExecutionStatus; -import com.vmware.taurus.service.model.ExecutionType; - -public class GraphQLJobExecutionsSortByEndTimeIT extends BaseDataJobDeploymentIT { - - @Autowired JobExecutionRepository jobExecutionRepository; - - @Autowired JobsRepository jobsRepository; - - @BeforeEach - public void cleanup() { - jobExecutionRepository.deleteAll(); - } - - private static String getQuery(String jobName, String executionsSortOrder) { - return "{\n" - + " jobs(pageNumber: 1, pageSize: 100, filter: [{property: \"jobName\", pattern: \"" - + jobName - + "\", sort: ASC}]) {\n" - + " content {\n" - + " jobName\n" - + " deployments {\n" - + " executions(pageNumber: 1, pageSize: 10, order: {property: \"endTime\"," - + " direction: " - + executionsSortOrder - + "}) {\n" - + " id\n" - + " endTime\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}"; - } - - @Test - public void testEmptyCallAsc(String jobName, String username) throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName, "ASC")) - .with(user(username))) - .andExpect(status().is(200)); - } - - @Test - public void testEmptyCallDesc(String jobName, String username) throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName, "DESC")) - .with(user(username))) - .andExpect(status().is(200)); - } - - @Test - public void testCallWithSingleExecution(String jobName, String username) throws Exception { - var expectedId = "testId1"; - var expectedEndTime = OffsetDateTime.now(); - - createDataJobExecution(expectedId, jobName, expectedEndTime); - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName, "DESC")) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId)) - .andExpect( - jsonPath("$.data.content[0].deployments[0].executions[0].endTime") - .value(expectedEndTime.toString())); - } - - @Test - public void testCallTwoExecutionsSortAsc(String jobName, String username) throws Exception { - var expectedId1 = "testId1"; - var expectedId2 = "testId2"; - var expectedEndTimeLarger = OffsetDateTime.now(); - var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); - - createDataJobExecution(expectedId1, jobName, expectedEndTimeLarger); - createDataJobExecution(expectedId2, jobName, expectedEndTimeSmaller); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName, "ASC")) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0]").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1]").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId2)) - .andExpect( - jsonPath("$.data.content[0].deployments[0].executions[0].endTime") - .value(expectedEndTimeSmaller.toString())) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1].id").value(expectedId1)) - .andExpect( - jsonPath("$.data.content[0].deployments[0].executions[1].endTime") - .value(expectedEndTimeLarger.toString())); - } - - @Test - public void testCallTwoExecutionsSortDesc(String jobName, String username) throws Exception { - var expectedId1 = "testId1"; - var expectedId2 = "testId2"; - var expectedEndTimeLarger = OffsetDateTime.now(); - var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); - - createDataJobExecution(expectedId1, jobName, expectedEndTimeLarger); - createDataJobExecution(expectedId2, jobName, expectedEndTimeSmaller); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName, "DESC")) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0]").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1]").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId1)) - .andExpect( - jsonPath("$.data.content[0].deployments[0].executions[0].endTime") - .value(expectedEndTimeLarger.toString())) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1].id").value(expectedId2)) - .andExpect( - jsonPath("$.data.content[0].deployments[0].executions[1].endTime") - .value(expectedEndTimeSmaller.toString())); - } - - @Test - public void testCallPagination(String jobName, String username) throws Exception { - for (int i = 0; i < 15; i++) { - createDataJobExecution(UUID.randomUUID().toString(), jobName, OffsetDateTime.now()); - } - // Query pagination is set to 10 items per page. - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName, "DESC")) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0]").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[9]").exists()) - .andExpect(jsonPath("$.data.content[0].deployments[0].executions[10]").doesNotExist()); - } - - private void createDataJobExecution(String executionId, String jobName, OffsetDateTime endTime) { - - DataJob dataJob = jobsRepository.findById(jobName).get(); - var jobExecution = - DataJobExecution.builder() - .id(executionId) - .dataJob(dataJob) - .startTime(OffsetDateTime.now()) - .endTime(endTime) - .type(ExecutionType.MANUAL) - .status(ExecutionStatus.SUCCEEDED) - .resourcesCpuRequest(1F) - .resourcesCpuLimit(2F) - .resourcesMemoryRequest(500) - .resourcesMemoryLimit(1000) - .message("message") - .lastDeployedBy("test_user") - .lastDeployedDate(OffsetDateTime.now()) - .jobVersion("test_version") - .jobSchedule("*/5 * * * *") - .opId("test_op_id") - .vdkVersion("test_vdk_version") - .build(); - - jobExecutionRepository.save(jobExecution); - } -} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java deleted file mode 100644 index 0328eca96f..0000000000 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright 2021 VMware, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.graphql.it; - -import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; -import com.vmware.taurus.service.JobExecutionRepository; -import com.vmware.taurus.service.JobsRepository; -import com.vmware.taurus.service.model.DataJob; -import com.vmware.taurus.service.model.DataJobExecution; -import com.vmware.taurus.service.model.ExecutionStatus; -import com.vmware.taurus.service.model.ExecutionType; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - -import java.time.OffsetDateTime; -import java.util.UUID; - -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -public class GraphQLJobExecutionsStatusCountIT extends BaseDataJobDeploymentIT { - - @Autowired JobExecutionRepository jobExecutionRepository; - - @Autowired JobsRepository jobsRepository; - - @BeforeEach - public void cleanup() { - jobExecutionRepository.deleteAll(); - } - - private void addJobExecution( - OffsetDateTime endTime, ExecutionStatus executionStatus, String jobName) { - var execution = - createDataJobExecution( - UUID.randomUUID().toString(), - jobName, - executionStatus, - "message", - OffsetDateTime.now()); - execution.setEndTime(endTime); - jobExecutionRepository.save(execution); - } - - private static String getQuery(String jobName) { - return "{\n" - + " jobs(pageNumber: 1, pageSize: 100, filter: [{property: \"jobName\", pattern: \"" - + jobName - + "\"}]) {\n" - + " content {\n" - + " jobName\n" - + " deployments {\n" - + " successfulExecutions\n " - + " failedExecutions\n " - + " }\n" - + " }\n" - + " }\n" - + "}"; - } - - @Test - public void testExecutionStatusCount_expectTwoSuccessful(String jobName, String username) - throws Exception { - var expectedEndTimeLarger = OffsetDateTime.now(); - var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); - - addJobExecution(expectedEndTimeLarger, ExecutionStatus.SUCCEEDED, jobName); - addJobExecution(expectedEndTimeSmaller, ExecutionStatus.SUCCEEDED, jobName); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName)) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(2)) - .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(0)); - } - - @Test - public void testExecutionStatusCount_expectNoCounts(String jobName, String username) - throws Exception { - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName)) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(0)) - .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(0)); - } - - @Test - public void testExecutionStatusCount_expectTwoSuccessfulTwoFailed(String jobName, String username) - throws Exception { - var expectedEndTimeLarger = OffsetDateTime.now(); - var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); - - addJobExecution(expectedEndTimeLarger, ExecutionStatus.SUCCEEDED, jobName); - addJobExecution(expectedEndTimeSmaller, ExecutionStatus.SUCCEEDED, jobName); - addJobExecution(expectedEndTimeLarger, ExecutionStatus.PLATFORM_ERROR, jobName); - addJobExecution(expectedEndTimeSmaller, ExecutionStatus.PLATFORM_ERROR, jobName); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName)) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(2)) - .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(2)); - } - - @Test - public void testExecutionStatusCount_expectOneSuccessfulOneFailed(String jobName, String username) - throws Exception { - var expectedEndTimeLarger = OffsetDateTime.now(); - - addJobExecution(expectedEndTimeLarger, ExecutionStatus.SUCCEEDED, jobName); - addJobExecution(expectedEndTimeLarger, ExecutionStatus.PLATFORM_ERROR, jobName); - - mockMvc - .perform( - MockMvcRequestBuilders.get(JOBS_URI) - .queryParam("query", getQuery(jobName)) - .with(user(username))) - .andExpect(status().is(200)) - .andExpect(content().contentType("application/json")) - .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(1)) - .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(1)); - } - - private DataJobExecution createDataJobExecution( - String executionId, - String jobName, - ExecutionStatus executionStatus, - String message, - OffsetDateTime startTime) { - - DataJob dataJob = jobsRepository.findById(jobName).get(); - - var jobExecution = - DataJobExecution.builder() - .id(executionId) - .dataJob(dataJob) - .startTime(startTime) - .type(ExecutionType.MANUAL) - .status(executionStatus) - .resourcesCpuRequest(1F) - .resourcesCpuLimit(2F) - .resourcesMemoryRequest(500) - .resourcesMemoryLimit(1000) - .message(message) - .lastDeployedBy("test_user") - .lastDeployedDate(OffsetDateTime.now()) - .jobVersion("test_version") - .jobSchedule("*/5 * * * *") - .opId("test_op_id") - .vdkVersion("test_vdk_version") - .build(); - - return jobExecution; - } -} From db5172d396a313524ce669258be0d172d58489bd Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 15:51:12 +0000 Subject: [PATCH 23/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../resources/application-private-builder.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties index 2a61b3c4ab..90a0abcea9 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -1,3 +1,3 @@ datajobs.builder.registrySecret=integration-test-docker-pull-secret -datajobs.builder.registrySecret.content.testOnly={BUILDER_TEST_REGISTRY_SECRET} +datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET} datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 From a5040495dadf670b2840d5e1e96dd973d25a292d Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 15:59:59 +0000 Subject: [PATCH 24/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../KerberosSecurityTestcaseJunit5.java | 25 +++++++++ .../common/MiniKdcCredentialsRepository.java | 53 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java new file mode 100644 index 0000000000..1697921b1d --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/KerberosSecurityTestcaseJunit5.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it.common; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.security.kerberos.test.KerberosSecurityTestcase; + +public class KerberosSecurityTestcaseJunit5 extends KerberosSecurityTestcase { + // TODO we should think about moving away from MiniKDC as it's not maintained. + @BeforeEach + @Override + public void startMiniKdc() throws Exception { + super.startMiniKdc(); + } + + @AfterEach + @Override + public void stopMiniKdc() { + super.stopMiniKdc(); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java new file mode 100644 index 0000000000..4850d95cea --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it.common; + +import com.vmware.taurus.service.credentials.KerberosCredentialsRepository; +import lombok.Setter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.kerberos.test.MiniKdc; + +import java.io.File; +import java.util.Optional; + +/** + * This class can be used to replace {@link KerberosCredentialsRepository} for testing purposes. It + * uses {@link MiniKdc} as a kerberos server. + * + *

TODO: Figure out how to connect to a testing KDC server via kadmin * For some reason I + * (tsvetkovt@vmware.com) was not able to connect to MiniKdc or to a * dockerized KDC with kadmin. + * TODO we should think about moving away from MiniKDC as it's not maintained. + */ +@Setter +public class MiniKdcCredentialsRepository extends KerberosCredentialsRepository { + private static final Logger log = LoggerFactory.getLogger(MiniKdcCredentialsRepository.class); + + private MiniKdc miniKdc; + + public MiniKdcCredentialsRepository() { + super("test", "test"); + } + + @Override + public void createPrincipal(String principal, Optional keytabLocation) { + try { + miniKdc.createPrincipal(keytabLocation.get(), principal); + } catch (Exception e) { + log.error("Failed to create principal", e); + } + } + + @Override + public boolean principalExists(String principal) { + return false; + } + + @Override + public void deletePrincipal(String principal) { + // Principals are deleted when MiniKdc is shut down. + } +} From c5c03c63ee644e46b94fca1df2b04c487a1e022c Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 16:01:20 +0000 Subject: [PATCH 25/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../java/com/vmware/taurus/datajobs/it/common/BaseIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 2e340372fc..d3f808b060 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -52,7 +52,6 @@ @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) @ExtendWith(WebHookServerMockExtension.class) public class BaseIT extends KerberosSecurityTestcaseJunit5 { - private static Logger log = LoggerFactory.getLogger(BaseIT.class); public static final String TEST_JOB_SCHEDULE = "15 10 * * *"; From 5abae3c6da203f486e7cb26532c03b9ad1c0784a Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 16:13:22 +0000 Subject: [PATCH 26/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/datajobs/it/common/MiniKdcCredentialsRepository.java | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java index 4850d95cea..2d3e3a710e 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java @@ -26,6 +26,7 @@ public class MiniKdcCredentialsRepository extends KerberosCredentialsRepository { private static final Logger log = LoggerFactory.getLogger(MiniKdcCredentialsRepository.class); + private MiniKdc miniKdc; public MiniKdcCredentialsRepository() { From ae2db2d7cfeeef75daac708698ef516c632155fe Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 16:13:48 +0000 Subject: [PATCH 27/69] Google Java Format --- .../taurus/datajobs/it/common/MiniKdcCredentialsRepository.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java index 2d3e3a710e..4850d95cea 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/MiniKdcCredentialsRepository.java @@ -26,7 +26,6 @@ public class MiniKdcCredentialsRepository extends KerberosCredentialsRepository { private static final Logger log = LoggerFactory.getLogger(MiniKdcCredentialsRepository.class); - private MiniKdc miniKdc; public MiniKdcCredentialsRepository() { From 4c925974567baa3e4dfba051b4e923721464501b Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 16:34:03 +0000 Subject: [PATCH 28/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../it/PrivateBuilderDockerRepoIT.java | 225 +-------- .../graphql/it/GraphQLDataJobsFieldsIT.java | 91 ++++ .../it/GraphQLDataJobsSortContactsIT.java | 190 ++++++++ .../graphql/it/GraphQLExecutionsIT.java | 430 ++++++++++++++++++ .../it/GraphQLExecutionsLogsUrlIT.java | 179 ++++++++ .../it/GraphQLExecutionsNextRunIT.java | 190 ++++++++ .../GraphQLJobExecutionsSortByEndTimeIT.java | 203 +++++++++ .../it/GraphQLJobExecutionsStatusCountIT.java | 174 +++++++ 8 files changed, 1479 insertions(+), 203 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 7ccd65c807..ecc1c53993 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -15,11 +15,13 @@ import com.vmware.taurus.controlplane.model.data.DataJobExecution; import com.vmware.taurus.controlplane.model.data.DataJobVersion; import com.vmware.taurus.datajobs.it.common.BaseIT; +import graphql.Assert; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1SecretBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; +import org.awaitility.Awaitility; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -198,210 +200,27 @@ public void testPrivateDockerBuildJob() throws Exception { .andReturn(); // wait for pod to initialize - Thread.sleep(10000); - - // retrieve running job execution id. - var exc = - mockMvc - .perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - var gson = new Gson(); - ArrayList parsed = - gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); - String executionId = (String) parsed.get(0).get("id"); - - // Check the data job execution status - checkDataJobExecutionStatus( - executionId, - DataJobExecution.StatusEnum.SUCCEEDED, - opId, - TEST_JOB_NAME, - TEST_TEAM_NAME, - "user"); - } - - private void checkDataJobExecutionStatus( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - try { - testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); - testDataJobDeploymentExecutionList( - executionId, executionStatus, opId, jobName, teamName, username); - testDataJobExecutionLogs(executionId, jobName, teamName, username); - } catch (Error e) { - try { - // print logs in case execution has failed - MvcResult dataJobExecutionLogsResult = - getExecuteLogs(executionId, jobName, teamName, username); - log.info( - "Job Execution {} logs:\n{}", - executionId, - dataJobExecutionLogsResult.getResponse().getContentAsString()); - } catch (Error ignore) { - } - throw e; - } - } - - private void testDataJobExecutionRead( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) { - - DataJobExecution[] dataJobExecution = new DataJobExecution[1]; - - await() - .atMost(5, TimeUnit.MINUTES) - .with() - .pollInterval(15, TimeUnit.SECONDS) - .until( - () -> { - String dataJobExecutionReadUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s", - teamName, jobName, executionId); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionReadUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - dataJobExecution[0] = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), - DataJobExecution.class); - if (dataJobExecution[0] == null) { - log.info("No response from server"); - } else { - log.info("Response from server " + dataJobExecution[0].getStatus()); + Thread.sleep(4000); + Awaitility.await().atMost(10 , TimeUnit.SECONDS) + .until(() -> { + try { + // retrieve running job execution id. + var exc = + mockMvc.perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + var gson = new Gson(); + ArrayList parsed = gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); + return StringUtils.isNotBlank((String) parsed.get(0).get("id")); + }catch(Exception e){ + return false; } - return dataJobExecution[0] != null - && executionStatus.equals(dataJobExecution[0].getStatus()); }); - - assertDataJobExecutionValid(executionId, executionStatus, opId, dataJobExecution[0], jobName); - } - - private void testDataJobExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobExecutionListUrl = - String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); - Assertions.assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - Assertions.assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); - } - - private void testDataJobDeploymentExecutionList( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - String jobName, - String teamName, - String username) - throws Exception { - - String dataJobDeploymentExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - teamName, jobName, "release"); - MvcResult dataJobExecutionResult = - mockMvc - .perform( - get(dataJobDeploymentExecutionListUrl) - .with(user(username)) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); - - List dataJobExecutions = - objectMapper.readValue( - dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); - Assertions.assertNotNull(dataJobExecutions); - dataJobExecutions = - dataJobExecutions.stream() - .filter(e -> e.getId().equals(executionId)) - .collect(Collectors.toList()); - Assertions.assertEquals(1, dataJobExecutions.size()); - assertDataJobExecutionValid( - executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); - } - - private void testDataJobExecutionLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); - Assertions.assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); - } - - @NotNull - private MvcResult getExecuteLogs( - String executionId, String jobName, String teamName, String username) throws Exception { - String dataJobExecutionListUrl = - String.format( - "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); - MvcResult dataJobExecutionLogsResult = - mockMvc - .perform(get(dataJobExecutionListUrl).with(user(username))) - .andExpect(status().isOk()) - .andReturn(); - return dataJobExecutionLogsResult; - } - - private void assertDataJobExecutionValid( - String executionId, - DataJobExecution.StatusEnum executionStatus, - String opId, - DataJobExecution dataJobExecution, - String jobName) { - - Assertions.assertNotNull(dataJobExecution); - Assertions.assertEquals(executionId, dataJobExecution.getId()); - Assertions.assertEquals(jobName, dataJobExecution.getJobName()); - Assertions.assertEquals(executionStatus, dataJobExecution.getStatus()); - Assertions.assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); - Assertions.assertEquals(opId, dataJobExecution.getOpId()); } } diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java new file mode 100644 index 0000000000..a3c1459272 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsFieldsIT.java @@ -0,0 +1,91 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.ExecutionStatus; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; + +import java.time.OffsetDateTime; +import java.time.ZoneOffset; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +public class GraphQLDataJobsFieldsIT extends BaseDataJobDeploymentIT { + + private static final String DEFAULT_QUERY_WITH_DEPLOYMENTS = + "query($filter: [Predicate], $executionOrder: DataJobExecutionOrder, $search: String," + + " $pageNumber: Int, $pageSize: Int) { jobs(pageNumber: $pageNumber, pageSize:" + + " $pageSize, filter: $filter, search: $search) { content { jobName " + + " deployments { id enabled lastExecutionStatus " + + " lastExecutionTime lastExecutionDuration executions(pageNumber: 1," + + " pageSize: 5, order: $executionOrder) { id status } } " + + " config { team description schedule { scheduleCron " + + " nextRunEpochSeconds } } } totalPages totalItems }}"; + + @Autowired private JobsRepository jobsRepository; + + @Test + void testFields(String jobName, String teamName, String username) throws Exception { + var dataJob = jobsRepository.findById(jobName).get(); + dataJob.setLastExecutionStatus(ExecutionStatus.SUCCEEDED); + dataJob.setLastExecutionEndTime(OffsetDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)); + dataJob.setLastExecutionDuration(1000); + dataJob = jobsRepository.save(dataJob); + + // Test requesting of fields that are computed + mockMvc + .perform( + get(JOBS_URI) + .with(user(username)) + .param("query", DEFAULT_QUERY_WITH_DEPLOYMENTS) + .param( + "variables", + "{" + + "\"search\": \"" + + jobName + + "\"," + + "\"pageNumber\": 1," + + "\"pageSize\": 10," + + "\"executionOrder\": {" + + " \"direction\": \"DESC\"," + + " \"property\": \"status\"" + + " }" + + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.content[0].config.team", is(teamName))) + .andExpect( + jsonPath( + "$.data.content[0].config.schedule.scheduleCron", + is(dataJob.getJobConfig().getSchedule()))) + .andExpect( + jsonPath("$.data.content[0].config.schedule.nextRunEpochSeconds", greaterThan(1))) + .andExpect( + jsonPath( + "$.data.content[0].deployments[0].lastExecutionStatus", + is(dataJob.getLastExecutionStatus().name()))) + .andExpect( + jsonPath( + "$.data.content[0].deployments[0].lastExecutionTime", + isDate(dataJob.getLastExecutionEndTime()))) + .andExpect( + jsonPath( + "$.data.content[0].deployments[0].lastExecutionDuration", + is(dataJob.getLastExecutionDuration()))) + .andReturn() + .getResponse() + .getContentAsString(); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java new file mode 100644 index 0000000000..f71204770d --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLDataJobsSortContactsIT.java @@ -0,0 +1,190 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.controlplane.model.data.DataJobContacts; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.JobConfig; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import java.util.List; + +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class GraphQLDataJobsSortContactsIT extends BaseIT { + + @Autowired private JobsRepository jobsRepository; + + @AfterEach + public void cleanup() { + jobsRepository.deleteAll(); + } + + private String getQuery(String sortOrder) { + return "{\n" + + " jobs(\n" + + " pageNumber: 1\n" + + " pageSize: 5\n" + + " filter: [{ property: \"config.contacts.present\", sort:" + + sortOrder + + "}]\n" + + " ) {\n" + + " content {\n" + + " jobName\n" + + " config {\n" + + " contacts{" + + " notifiedOnJobFailureUserError " + + " notifiedOnJobSuccess" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + @Test + public void testEmptyCall_shouldBeEmpty() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("DESC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content").isEmpty()); + } + + @Test + public void testSingleJob_shouldReturnOne() throws Exception { + var contacts = new DataJobContacts(); + contacts.setNotifiedOnJobFailureUserError(List.of("test-notified")); + var job = createDummyJob(contacts, "test-job"); + jobsRepository.save(job); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("DESC")) + .with(user(TEST_USERNAME))) + .andExpect( + jsonPath("$.data.content[0].config.contacts.notifiedOnJobFailureUserError") + .value("test-notified")); + } + + @Test + public void testSortingTwoJobs_shouldReturnDescOrder() throws Exception { + var contacts = new DataJobContacts(); + contacts.setNotifiedOnJobFailureUserError(List.of("test-notified")); + var job = createDummyJob(contacts, "test-job"); + + var contacts2 = new DataJobContacts(); + var job2 = createDummyJob(contacts2, "test-job2"); + + jobsRepository.save(job); + jobsRepository.save(job2); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("DESC")) + .with(user(TEST_USERNAME))) + .andExpect( + jsonPath("$.data.content[0].config.contacts.notifiedOnJobFailureUserError") + .value("test-notified")) + .andExpect( + jsonPath("$.data.content[1].config.contacts.notifiedOnJobFailureUserError").isEmpty()); + } + + @Test + public void testSortingTwoJobs_shouldReturnAscOrder() throws Exception { + var contacts = new DataJobContacts(); + contacts.setNotifiedOnJobSuccess(List.of("test-notified")); + var job = createDummyJob(contacts, "test-job"); + + var contacts2 = new DataJobContacts(); + var job2 = createDummyJob(contacts2, "test-job2"); + + jobsRepository.save(job); + jobsRepository.save(job2); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("ASC")) + .with(user(TEST_USERNAME))) + .andExpect( + jsonPath("$.data.content[1].config.contacts.notifiedOnJobSuccess") + .value("test-notified")) + .andExpect(jsonPath("$.data.content[0].config.contacts.notifiedOnJobSuccess").isEmpty()); + } + + @Test + public void testSortingTwoJobsNoContacts_shouldReturnTwo() throws Exception { + var contacts = new DataJobContacts(); + var job = createDummyJob(contacts, "test-job"); + + var contacts2 = new DataJobContacts(); + var job2 = createDummyJob(contacts2, "test-job2"); + + jobsRepository.save(job); + jobsRepository.save(job2); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("ASC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content.length()").value(2)) + .andExpect( + jsonPath("$.data.content[*].jobName", Matchers.contains("test-job", "test-job2"))); + } + + @Test + public void testSortingTwoJobsWithContacts_shouldReturnTwo() throws Exception { + var contacts = new DataJobContacts(); + contacts.setNotifiedOnJobDeploy(List.of("test")); + var job = createDummyJob(contacts, "test-job"); + + var contacts2 = new DataJobContacts(); + contacts2.setNotifiedOnJobFailurePlatformError(List.of("test")); + var job2 = createDummyJob(contacts2, "test-job2"); + + jobsRepository.save(job); + jobsRepository.save(job2); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("ASC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content.length()").value(2)) + .andExpect( + jsonPath("$.data.content[*].jobName", Matchers.contains("test-job", "test-job2"))); + } + + private DataJob createDummyJob(DataJobContacts contacts, String id) { + DataJob job = new DataJob(); + job.setName(id); + JobConfig config = new JobConfig(); + config.setSchedule("test-schedule"); + config.setNotifiedOnJobFailureUserError(contacts.getNotifiedOnJobFailureUserError()); + config.setNotifiedOnJobDeploy(contacts.getNotifiedOnJobDeploy()); + config.setNotifiedOnJobSuccess(contacts.getNotifiedOnJobSuccess()); + config.setNotifiedOnJobFailurePlatformError(contacts.getNotifiedOnJobFailurePlatformError()); + job.setJobConfig(config); + return job; + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java new file mode 100644 index 0000000000..1cde318a6c --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java @@ -0,0 +1,430 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.datajobs.it.common.JobExecutionUtil; +import com.vmware.taurus.service.JobExecutionRepository; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.DataJobExecution; +import com.vmware.taurus.service.model.ExecutionStatus; +import com.vmware.taurus.service.model.JobConfig; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class GraphQLExecutionsIT extends BaseIT { + + @Autowired JobExecutionRepository jobExecutionRepository; + + @Autowired JobsRepository jobsRepository; + + private static final String TEST_JOB_NAME_1 = "TEST-JOB-NAME-1"; + private static final String TEST_JOB_NAME_2 = "TEST-JOB-NAME-2"; + private static final String TEST_JOB_NAME_3 = "TEST-JOB-NAME-3"; + + private DataJob dataJob1; + private DataJob dataJob2; + private DataJob dataJob3; + + private DataJobExecution dataJobExecution1; + private DataJobExecution dataJobExecution2; + private DataJobExecution dataJobExecution3; + + @BeforeEach + public void setup() { + cleanup(); + + var config1 = new JobConfig(); + config1.setTeam("test-team1"); + + var config2 = new JobConfig(); + config2.setTeam("test-team2"); + + var config3 = new JobConfig(); + config3.setTeam("test-team3"); + + this.dataJob1 = jobsRepository.save(new DataJob(TEST_JOB_NAME_1, config1)); + this.dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, config2)); + this.dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, config3)); + + OffsetDateTime now = OffsetDateTime.now(); + this.dataJobExecution1 = + JobExecutionUtil.createDataJobExecution( + jobExecutionRepository, "testId1", dataJob1, now, now, ExecutionStatus.SUCCEEDED); + this.dataJobExecution2 = + JobExecutionUtil.createDataJobExecution( + jobExecutionRepository, + "testId2", + dataJob2, + now.minusSeconds(1), + now.minusSeconds(1), + ExecutionStatus.USER_ERROR); + this.dataJobExecution3 = + JobExecutionUtil.createDataJobExecution( + jobExecutionRepository, + "testId3", + dataJob3, + now.minusSeconds(10), + now.minusSeconds(10), + ExecutionStatus.SUBMITTED); + } + + private static String getQuery() { + return "query($filter: DataJobExecutionFilter, $order: DataJobExecutionOrder, $pageNumber: Int," + + " $pageSize: Int) { executions(pageNumber: $pageNumber, pageSize: $pageSize," + + " filter: $filter, order: $order) { content { id jobName " + + " startTime endTime status deployment { id enabled " + + " jobVersion deployedDate deployedBy resources { " + + " cpuLimit cpuRequest memoryLimit memoryRequest " + + " } schedule { scheduleCron } vdkVersion } " + + " } totalPages totalItems }}"; + } + + private void cleanup() { + jobsRepository + .findAllById(List.of(TEST_JOB_NAME_1, TEST_JOB_NAME_2, TEST_JOB_NAME_3)) + .forEach(dataJob -> jobsRepository.delete(dataJob)); + } + + @Test + public void testExecutions_filterByStartTimeGte_shouldReturnAllProperties() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"startTimeGte\": \"" + + dataJobExecution2.getStartTime() + + "\"" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user(TEST_USERNAME))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.contains(dataJobExecution1.getId(), dataJobExecution2.getId()))) + .andExpect( + jsonPath( + "$.data.content[*].jobName", + Matchers.contains(dataJob1.getName(), dataJob2.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains( + dataJobExecution1.getStatus().toString(), + dataJobExecution2.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.jobVersion", + Matchers.contains( + dataJobExecution1.getJobVersion(), dataJobExecution2.getJobVersion()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.deployedDate", + Matchers.contains( + dataJobExecution1.getLastDeployedDate().toString(), + dataJobExecution2.getLastDeployedDate().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.deployedBy", + Matchers.contains( + dataJobExecution1.getLastDeployedBy(), dataJobExecution2.getLastDeployedBy()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.vdkVersion", + Matchers.contains( + dataJobExecution1.getVdkVersion(), dataJobExecution2.getVdkVersion()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.resources.cpuRequest", + Matchers.contains( + dataJobExecution1.getResourcesCpuRequest().doubleValue(), + dataJobExecution2.getResourcesCpuRequest().doubleValue()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.resources.cpuLimit", + Matchers.contains( + dataJobExecution1.getResourcesCpuLimit().doubleValue(), + dataJobExecution2.getResourcesCpuLimit().doubleValue()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.resources.memoryRequest", + Matchers.contains( + dataJobExecution1.getResourcesMemoryRequest(), + dataJobExecution2.getResourcesMemoryRequest()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.resources.memoryLimit", + Matchers.contains( + dataJobExecution1.getResourcesMemoryLimit(), + dataJobExecution2.getResourcesMemoryLimit()))) + .andExpect( + jsonPath( + "$.data.content[*].deployment.schedule.scheduleCron", + Matchers.contains( + dataJobExecution1.getJobSchedule(), dataJobExecution2.getJobSchedule()))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution3.getId())))); + } + + @Test + public void testExecutions_filterByStartTimeLte() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"startTimeLte\": \"" + + dataJobExecution2.getStartTime() + + "\"" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user(TEST_USERNAME))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.contains(dataJobExecution2.getId(), dataJobExecution3.getId()))) + .andExpect( + jsonPath( + "$.data.content[*].jobName", + Matchers.contains(dataJob2.getName(), dataJob3.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains( + dataJobExecution2.getStatus().toString(), + dataJobExecution3.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution1.getId())))); + } + + @Test + public void testExecutions_filterByEndTimeGte() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"endTimeGte\": \"" + + dataJobExecution2.getEndTime() + + "\"" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user("user"))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.contains(dataJobExecution1.getId(), dataJobExecution2.getId()))) + .andExpect( + jsonPath( + "$.data.content[*].jobName", + Matchers.contains(dataJob1.getName(), dataJob2.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains( + dataJobExecution1.getStatus().toString(), + dataJobExecution2.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution3.getId())))); + } + + @Test + public void testExecutions_filterByEndTimeLte() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"endTimeLte\": \"" + + dataJobExecution2.getEndTime() + + "\"" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user("user"))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.contains(dataJobExecution2.getId(), dataJobExecution3.getId()))) + .andExpect( + jsonPath( + "$.data.content[*].jobName", + Matchers.contains(dataJob2.getName(), dataJob3.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains( + dataJobExecution2.getStatus().toString(), + dataJobExecution3.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution1.getId())))); + } + + @Test + public void testExecutions_filterByStatusIn() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"statusIn\": [\"" + + dataJobExecution1.getStatus().toString() + + "\", \"" + + dataJobExecution2.getStatus().toString() + + "\"]" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user("user"))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.contains(dataJobExecution1.getId(), dataJobExecution2.getId()))) + .andExpect( + jsonPath( + "$.data.content[*].jobName", + Matchers.contains(dataJob1.getName(), dataJob2.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains( + dataJobExecution1.getStatus().toString(), + dataJobExecution2.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution3.getId())))); + } + + @Test + public void testExecutions_filterByJobNameIn() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"jobNameIn\": [\"" + + dataJobExecution1.getDataJob().getName() + + "\"]" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user("user"))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[*].id", Matchers.contains(dataJobExecution1.getId()))) + .andExpect(jsonPath("$.data.content[*].jobName", Matchers.contains(dataJob1.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains(dataJobExecution1.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution3.getId())))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution2.getId())))); + } + + @Test + public void testExecutions_filterByTeamNameIn() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"teamNameIn\": [\"" + + dataJobExecution1.getDataJob().getJobConfig().getTeam() + + "\"]" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user("user"))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[*].id", Matchers.contains(dataJobExecution1.getId()))) + .andExpect(jsonPath("$.data.content[*].jobName", Matchers.contains(dataJob1.getName()))) + .andExpect( + jsonPath( + "$.data.content[*].status", + Matchers.contains(dataJobExecution1.getStatus().toString()))) + .andExpect( + jsonPath( + "$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution3.getId())))) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.not(Matchers.contains(dataJobExecution2.getId())))); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java new file mode 100644 index 0000000000..b073ff07bc --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsLogsUrlIT.java @@ -0,0 +1,179 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.text.MessageFormat; +import java.time.OffsetDateTime; +import java.util.List; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.datajobs.it.common.JobExecutionUtil; +import com.vmware.taurus.service.JobExecutionRepository; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.DataJobExecution; +import com.vmware.taurus.service.model.ExecutionStatus; +import com.vmware.taurus.service.model.JobConfig; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class GraphQLExecutionsLogsUrlIT extends BaseIT { + + @Autowired JobExecutionRepository jobExecutionRepository; + + @Autowired JobsRepository jobsRepository; + + private static final String LOGS_URL_TEMPLATE = + "https://log-insight-base-url/li/query/stream?query=%C2%A7%C2%A7%C2%A7AND%C" + + "2%A7%C2%A7%C2%A7%C2%A7{0}%C2%A7{1}%C2%A7true%C2%A7COUNT%C2%A7*%C2%A7" + + "timestamp%C2%A7pageSortPreference:%7B%22sortBy%22%3A%22-{2}-{3}-" + + "ingest_timestamp%22%2C%22sortOrder%22%3A%22DESC%22%7D%C2%A7" + + "alertDefList:%5B%5D%C2%A7partitions:%C2%A7%C2%A7text:CONTAINS:{4}*"; + + private static final String TEST_JOB_NAME_1 = "TEST-JOB-NAME-1"; + private static final String TEST_JOB_NAME_2 = "TEST-JOB-NAME-2"; + private static final String TEST_JOB_NAME_3 = "TEST-JOB-NAME-3"; + + private static final long TEST_START_TIME_OFFSET_SECONDS = 1000; + private static final long TEST_END_TIME_OFFSET_SECONDS = -1000; + + private DataJobExecution dataJobExecution1; + private DataJobExecution dataJobExecution2; + private DataJobExecution dataJobExecution3; + + @DynamicPropertySource + static void dynamicProperties(DynamicPropertyRegistry registry) { + String logsUrlTemplate = + MessageFormat.format( + LOGS_URL_TEMPLATE, + "{{start_time}}", + "{{end_time}}", + "{{job_name}}", + "{{op_id}}", + "{{execution_id}}"); + registry.add("datajobs.executions.logsUrl.template", () -> logsUrlTemplate); + registry.add("datajobs.executions.logsUrl.dateFormat", () -> "unix"); + registry.add( + "datajobs.executions.logsUrl.startTimeOffsetSeconds", () -> TEST_START_TIME_OFFSET_SECONDS); + registry.add( + "datajobs.executions.logsUrl.endTimeOffsetSeconds", () -> TEST_END_TIME_OFFSET_SECONDS); + } + + @BeforeEach + public void setup() { + cleanup(); + + DataJob dataJob1 = jobsRepository.save(new DataJob(TEST_JOB_NAME_1, new JobConfig())); + DataJob dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, new JobConfig())); + DataJob dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, new JobConfig())); + + OffsetDateTime now = OffsetDateTime.now(); + this.dataJobExecution1 = + JobExecutionUtil.createDataJobExecution( + jobExecutionRepository, "testId1", dataJob1, now, now, ExecutionStatus.SUCCEEDED); + this.dataJobExecution2 = + JobExecutionUtil.createDataJobExecution( + jobExecutionRepository, + "testId2", + dataJob2, + now.minusSeconds(1), + now.minusSeconds(1), + ExecutionStatus.RUNNING); + this.dataJobExecution3 = + JobExecutionUtil.createDataJobExecution( + jobExecutionRepository, + "testId3", + dataJob3, + now.minusSeconds(10), + now.minusSeconds(10), + ExecutionStatus.SUBMITTED); + } + + private static String getQuery() { + return "query($filter: DataJobExecutionFilter, $order: DataJobExecutionOrder, $pageNumber: Int," + + " $pageSize: Int) { executions(pageNumber: $pageNumber, pageSize: $pageSize," + + " filter: $filter, order: $order) { content { id logsUrl } " + + " totalPages totalItems }}"; + } + + private void cleanup() { + jobsRepository + .findAllById(List.of(TEST_JOB_NAME_1, TEST_JOB_NAME_2, TEST_JOB_NAME_3)) + .forEach(dataJob -> jobsRepository.delete(dataJob)); + } + + @Test + public void testExecutions_filterByStartTimeGte() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery()) + .param( + "variables", + "{" + + "\"filter\": {" + + " \"startTimeGte\": \"" + + dataJobExecution3.getStartTime() + + "\"" + + " }," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .with(user(TEST_USERNAME))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect( + jsonPath( + "$.data.content[*].id", + Matchers.contains( + dataJobExecution1.getId(), + dataJobExecution2.getId(), + dataJobExecution3.getId()))) + .andExpect( + jsonPath( + "$.data.content[*].logsUrl", + Matchers.contains( + buildLogsUrl(dataJobExecution1), + buildLogsUrl(dataJobExecution2), + buildLogsUrl(dataJobExecution3)))); + } + + private static String buildLogsUrl(DataJobExecution dataJobExecution) { + return MessageFormat.format( + LOGS_URL_TEMPLATE, + String.valueOf( + dataJobExecution + .getStartTime() + .toInstant() + .plusSeconds(TEST_START_TIME_OFFSET_SECONDS) + .toEpochMilli()), + String.valueOf( + dataJobExecution + .getEndTime() + .toInstant() + .plusSeconds(TEST_END_TIME_OFFSET_SECONDS) + .toEpochMilli()), + dataJobExecution.getDataJob().getName(), + dataJobExecution.getOpId(), + dataJobExecution.getId()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java new file mode 100644 index 0000000000..7757d33a61 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsNextRunIT.java @@ -0,0 +1,190 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.DeploymentStatus; +import com.vmware.taurus.service.model.JobConfig; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import java.time.OffsetDateTime; +import java.time.ZoneId; + +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class GraphQLExecutionsNextRunIT extends BaseIT { + + @Autowired JobsRepository jobsRepository; + + @BeforeEach + public void cleanup() { + jobsRepository.deleteAll(); + } + + private String getQuery(String sortOrder) { + return "{\n" + + " jobs(pageNumber: 1, pageSize: 100, filter: [{property:" + + " \"config.schedule.nextRunEpochSeconds\", sort:" + + sortOrder + + "}]) {\n" + + " content {\n" + + " jobName \n" + + " config {\n" + + " schedule {\n" + + " scheduleCron\n" + + " nextRunEpochSeconds\n" + + " }" + + " }" + + " }\n" + + " }\n" + + "}"; + } + + private String getQueryWithFilter(String filter) { + return "{\n" + + " jobs(pageNumber: 1, pageSize: 100, filter: [{property:" + + " \"config.schedule.nextRunEpochSeconds\", pattern:" + + filter + + "}]) {\n" + + " content {\n" + + " jobName \n" + + " config {\n" + + " schedule {\n" + + " scheduleCron\n" + + " nextRunEpochSeconds\n" + + " }" + + " }" + + " }\n" + + " }\n" + + "}"; + } + + @Test + public void testNextRunCall_noJobs() throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("DESC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content").isEmpty()); + } + + @Test + public void testNextRunCall_oneJob() throws Exception { + var now = OffsetDateTime.now(); + addJob("jobA", toCron(now)); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("DESC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content[0].jobName").value("jobA")) + .andExpect(jsonPath("$.data.content[0].config.schedule.scheduleCron").value(toCron(now))); + } + + @Test + public void testNextRunCall_twoJobs_DESC() throws Exception { + var now = OffsetDateTime.now(ZoneId.of("UTC")).plusMinutes(2); + var later = now.plusDays(1); + + addJob("jobA", toCron(now)); + addJob("jobB", toCron(later)); + // check correct sort is applied + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("DESC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content[0].jobName").value("jobB")) + .andExpect(jsonPath("$.data.content[1].jobName").value("jobA")) + .andExpect(jsonPath("$.data.content[0].config.schedule.scheduleCron").value(toCron(later))) + .andExpect(jsonPath("$.data.content[1].config.schedule.scheduleCron").value(toCron(now))); + } + + @Test + public void testNextRunCall_twoJobs_ASC() throws Exception { + var now = OffsetDateTime.now(ZoneId.of("UTC")).plusMinutes(2); + var later = now.plusMinutes(60); + + addJob("jobA", toCron(now)); + addJob("jobB", toCron(later)); + // check correct sort is applied + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery("ASC")) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content[0].jobName").value("jobA")) + .andExpect(jsonPath("$.data.content[1].jobName").value("jobB")) + .andExpect(jsonPath("$.data.content[0].config.schedule.scheduleCron").value(toCron(now))) + .andExpect(jsonPath("$.data.content[1].config.schedule.scheduleCron").value(toCron(later))); + } + + @Test + public void testNextRunCall_filter_noJobs() throws Exception { + var now = OffsetDateTime.now(); + var before = now.minusDays(2); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQueryWithFilter(getFilterPattern(before, now))) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content").isEmpty()); + } + + @Test + public void testNextRunCall_filter_twoJobs() throws Exception { + var now = OffsetDateTime.now(ZoneId.of("UTC")); + var after = now.plusMinutes(120); + + addJob("jobA", toCron(now.minusMinutes(240))); + addJob("jobB", toCron(now.plusMinutes(30))); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQueryWithFilter(getFilterPattern(now, after))) + .with(user(TEST_USERNAME))) + .andExpect(jsonPath("$.data.content[0].jobName").value("jobB")) + .andExpect(jsonPath("$.data.content[*].jobName", Matchers.not(Matchers.contains("jobA")))); + } + + private void addJob(String jobName, String jobSchedule) { + JobConfig config = new JobConfig(); + config.setSchedule(jobSchedule); + var job = new DataJob(jobName, config); + job.setLatestJobDeploymentStatus(DeploymentStatus.SUCCESS); + jobsRepository.save(job); + } + + private String toCron(OffsetDateTime dateTime) { + return String.format( + "%d %d %d %d %d", + dateTime.getMinute(), + dateTime.getHour(), + dateTime.getDayOfMonth(), + dateTime.getMonth().getValue(), + dateTime.getDayOfWeek().getValue()); + } + + private String getFilterPattern(OffsetDateTime start, OffsetDateTime finish) { + return String.format("\"%d-%d\"", start.toEpochSecond(), finish.toEpochSecond()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java new file mode 100644 index 0000000000..7f197bae63 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsSortByEndTimeIT.java @@ -0,0 +1,203 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.time.OffsetDateTime; +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; +import com.vmware.taurus.service.JobExecutionRepository; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.DataJobExecution; +import com.vmware.taurus.service.model.ExecutionStatus; +import com.vmware.taurus.service.model.ExecutionType; + +public class GraphQLJobExecutionsSortByEndTimeIT extends BaseDataJobDeploymentIT { + + @Autowired JobExecutionRepository jobExecutionRepository; + + @Autowired JobsRepository jobsRepository; + + @BeforeEach + public void cleanup() { + jobExecutionRepository.deleteAll(); + } + + private static String getQuery(String jobName, String executionsSortOrder) { + return "{\n" + + " jobs(pageNumber: 1, pageSize: 100, filter: [{property: \"jobName\", pattern: \"" + + jobName + + "\", sort: ASC}]) {\n" + + " content {\n" + + " jobName\n" + + " deployments {\n" + + " executions(pageNumber: 1, pageSize: 10, order: {property: \"endTime\"," + + " direction: " + + executionsSortOrder + + "}) {\n" + + " id\n" + + " endTime\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + @Test + public void testEmptyCallAsc(String jobName, String username) throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName, "ASC")) + .with(user(username))) + .andExpect(status().is(200)); + } + + @Test + public void testEmptyCallDesc(String jobName, String username) throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName, "DESC")) + .with(user(username))) + .andExpect(status().is(200)); + } + + @Test + public void testCallWithSingleExecution(String jobName, String username) throws Exception { + var expectedId = "testId1"; + var expectedEndTime = OffsetDateTime.now(); + + createDataJobExecution(expectedId, jobName, expectedEndTime); + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName, "DESC")) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId)) + .andExpect( + jsonPath("$.data.content[0].deployments[0].executions[0].endTime") + .value(expectedEndTime.toString())); + } + + @Test + public void testCallTwoExecutionsSortAsc(String jobName, String username) throws Exception { + var expectedId1 = "testId1"; + var expectedId2 = "testId2"; + var expectedEndTimeLarger = OffsetDateTime.now(); + var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); + + createDataJobExecution(expectedId1, jobName, expectedEndTimeLarger); + createDataJobExecution(expectedId2, jobName, expectedEndTimeSmaller); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName, "ASC")) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0]").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1]").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId2)) + .andExpect( + jsonPath("$.data.content[0].deployments[0].executions[0].endTime") + .value(expectedEndTimeSmaller.toString())) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1].id").value(expectedId1)) + .andExpect( + jsonPath("$.data.content[0].deployments[0].executions[1].endTime") + .value(expectedEndTimeLarger.toString())); + } + + @Test + public void testCallTwoExecutionsSortDesc(String jobName, String username) throws Exception { + var expectedId1 = "testId1"; + var expectedId2 = "testId2"; + var expectedEndTimeLarger = OffsetDateTime.now(); + var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); + + createDataJobExecution(expectedId1, jobName, expectedEndTimeLarger); + createDataJobExecution(expectedId2, jobName, expectedEndTimeSmaller); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName, "DESC")) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0]").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1]").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0].id").value(expectedId1)) + .andExpect( + jsonPath("$.data.content[0].deployments[0].executions[0].endTime") + .value(expectedEndTimeLarger.toString())) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[1].id").value(expectedId2)) + .andExpect( + jsonPath("$.data.content[0].deployments[0].executions[1].endTime") + .value(expectedEndTimeSmaller.toString())); + } + + @Test + public void testCallPagination(String jobName, String username) throws Exception { + for (int i = 0; i < 15; i++) { + createDataJobExecution(UUID.randomUUID().toString(), jobName, OffsetDateTime.now()); + } + // Query pagination is set to 10 items per page. + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName, "DESC")) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[0]").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[9]").exists()) + .andExpect(jsonPath("$.data.content[0].deployments[0].executions[10]").doesNotExist()); + } + + private void createDataJobExecution(String executionId, String jobName, OffsetDateTime endTime) { + + DataJob dataJob = jobsRepository.findById(jobName).get(); + var jobExecution = + DataJobExecution.builder() + .id(executionId) + .dataJob(dataJob) + .startTime(OffsetDateTime.now()) + .endTime(endTime) + .type(ExecutionType.MANUAL) + .status(ExecutionStatus.SUCCEEDED) + .resourcesCpuRequest(1F) + .resourcesCpuLimit(2F) + .resourcesMemoryRequest(500) + .resourcesMemoryLimit(1000) + .message("message") + .lastDeployedBy("test_user") + .lastDeployedDate(OffsetDateTime.now()) + .jobVersion("test_version") + .jobSchedule("*/5 * * * *") + .opId("test_op_id") + .vdkVersion("test_vdk_version") + .build(); + + jobExecutionRepository.save(jobExecution); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java new file mode 100644 index 0000000000..0328eca96f --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java @@ -0,0 +1,174 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.graphql.it; + +import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; +import com.vmware.taurus.service.JobExecutionRepository; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.DataJobExecution; +import com.vmware.taurus.service.model.ExecutionStatus; +import com.vmware.taurus.service.model.ExecutionType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import java.time.OffsetDateTime; +import java.util.UUID; + +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +public class GraphQLJobExecutionsStatusCountIT extends BaseDataJobDeploymentIT { + + @Autowired JobExecutionRepository jobExecutionRepository; + + @Autowired JobsRepository jobsRepository; + + @BeforeEach + public void cleanup() { + jobExecutionRepository.deleteAll(); + } + + private void addJobExecution( + OffsetDateTime endTime, ExecutionStatus executionStatus, String jobName) { + var execution = + createDataJobExecution( + UUID.randomUUID().toString(), + jobName, + executionStatus, + "message", + OffsetDateTime.now()); + execution.setEndTime(endTime); + jobExecutionRepository.save(execution); + } + + private static String getQuery(String jobName) { + return "{\n" + + " jobs(pageNumber: 1, pageSize: 100, filter: [{property: \"jobName\", pattern: \"" + + jobName + + "\"}]) {\n" + + " content {\n" + + " jobName\n" + + " deployments {\n" + + " successfulExecutions\n " + + " failedExecutions\n " + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + @Test + public void testExecutionStatusCount_expectTwoSuccessful(String jobName, String username) + throws Exception { + var expectedEndTimeLarger = OffsetDateTime.now(); + var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); + + addJobExecution(expectedEndTimeLarger, ExecutionStatus.SUCCEEDED, jobName); + addJobExecution(expectedEndTimeSmaller, ExecutionStatus.SUCCEEDED, jobName); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName)) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(2)) + .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(0)); + } + + @Test + public void testExecutionStatusCount_expectNoCounts(String jobName, String username) + throws Exception { + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName)) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(0)) + .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(0)); + } + + @Test + public void testExecutionStatusCount_expectTwoSuccessfulTwoFailed(String jobName, String username) + throws Exception { + var expectedEndTimeLarger = OffsetDateTime.now(); + var expectedEndTimeSmaller = OffsetDateTime.now().minusDays(1); + + addJobExecution(expectedEndTimeLarger, ExecutionStatus.SUCCEEDED, jobName); + addJobExecution(expectedEndTimeSmaller, ExecutionStatus.SUCCEEDED, jobName); + addJobExecution(expectedEndTimeLarger, ExecutionStatus.PLATFORM_ERROR, jobName); + addJobExecution(expectedEndTimeSmaller, ExecutionStatus.PLATFORM_ERROR, jobName); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName)) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(2)) + .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(2)); + } + + @Test + public void testExecutionStatusCount_expectOneSuccessfulOneFailed(String jobName, String username) + throws Exception { + var expectedEndTimeLarger = OffsetDateTime.now(); + + addJobExecution(expectedEndTimeLarger, ExecutionStatus.SUCCEEDED, jobName); + addJobExecution(expectedEndTimeLarger, ExecutionStatus.PLATFORM_ERROR, jobName); + + mockMvc + .perform( + MockMvcRequestBuilders.get(JOBS_URI) + .queryParam("query", getQuery(jobName)) + .with(user(username))) + .andExpect(status().is(200)) + .andExpect(content().contentType("application/json")) + .andExpect(jsonPath("$.data.content[0].deployments[0].successfulExecutions").value(1)) + .andExpect(jsonPath("$.data.content[0].deployments[0].failedExecutions").value(1)); + } + + private DataJobExecution createDataJobExecution( + String executionId, + String jobName, + ExecutionStatus executionStatus, + String message, + OffsetDateTime startTime) { + + DataJob dataJob = jobsRepository.findById(jobName).get(); + + var jobExecution = + DataJobExecution.builder() + .id(executionId) + .dataJob(dataJob) + .startTime(startTime) + .type(ExecutionType.MANUAL) + .status(executionStatus) + .resourcesCpuRequest(1F) + .resourcesCpuLimit(2F) + .resourcesMemoryRequest(500) + .resourcesMemoryLimit(1000) + .message(message) + .lastDeployedBy("test_user") + .lastDeployedDate(OffsetDateTime.now()) + .jobVersion("test_version") + .jobSchedule("*/5 * * * *") + .opId("test_op_id") + .vdkVersion("test_vdk_version") + .build(); + + return jobExecution; + } +} From 9138171f3147925087dec86b158012e196e0eb81 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 16:34:36 +0000 Subject: [PATCH 29/69] Google Java Format --- .../it/PrivateBuilderDockerRepoIT.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index ecc1c53993..9437913e45 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -6,23 +6,19 @@ package com.vmware.taurus.datajobs.it; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.gson.Gson; import com.google.gson.internal.LinkedTreeMap; import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.controlplane.model.data.DataJobExecution; import com.vmware.taurus.controlplane.model.data.DataJobVersion; import com.vmware.taurus.datajobs.it.common.BaseIT; -import graphql.Assert; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1SecretBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.awaitility.Awaitility; -import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -37,14 +33,11 @@ import org.springframework.test.web.servlet.MvcResult; import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; -import static org.awaitility.Awaitility.await; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -201,24 +194,28 @@ public void testPrivateDockerBuildJob() throws Exception { // wait for pod to initialize Thread.sleep(4000); - Awaitility.await().atMost(10 , TimeUnit.SECONDS) - .until(() -> { + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until( + () -> { try { // retrieve running job execution id. var exc = - mockMvc.perform( - get(String.format( - "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", - TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) - .with(user("user")) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andReturn(); + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); var gson = new Gson(); - ArrayList parsed = gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); + ArrayList parsed = + gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); return StringUtils.isNotBlank((String) parsed.get(0).get("id")); - }catch(Exception e){ + } catch (Exception e) { return false; } }); From c25c5752f6a5a976c4cbf63ff31e5fe76d5ea721 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 16:36:03 +0000 Subject: [PATCH 30/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../datajobs/it/DataJobCancellationIT.java | 200 +++++++++ .../it/DataJobEphemeralStorageIT.java | 369 +++++++++++++++++ .../taurus/datajobs/it/DataJobGraphQLIT.java | 293 +++++++++++++ .../datajobs/it/DataJobPropertiesIT.java | 88 ++++ .../it/DataJobTerminationStatusIT.java | 386 ++++++++++++++++++ .../it/common/BaseDataJobDeploymentIT.java | 88 ++++ 6 files changed, 1424 insertions(+) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java new file mode 100644 index 0000000000..4ac41b3390 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCancellationIT.java @@ -0,0 +1,200 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.controlplane.model.data.DataJobVersion; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; + +import java.util.ArrayList; +import java.util.UUID; + +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@Slf4j +@Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class}) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class DataJobCancellationIT extends BaseIT { + + private static final String TEST_JOB_NAME = + "cancellation-test-" + UUID.randomUUID().toString().substring(0, 8); + private static final Object DEPLOYMENT_ID = "testing-cancellation"; + + @AfterEach + public void cleanUp() throws Exception { + // delete job + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isOk()); + + // Execute delete deployment + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + } + + @BeforeEach + public void setup() throws Exception { + String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + // Execute create job + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()) + .andExpect( + header() + .string( + HttpHeaders.LOCATION, + lambdaMatcher( + s -> + s.endsWith( + String.format( + "/data-jobs/for-team/%s/jobs/%s", + TEST_TEAM_NAME, TEST_JOB_NAME))))); + } + + @Test + public void testJobCancellation_createDeployExecuteAndCancelJob() throws Exception { + // Take the job zip as byte array + byte[] jobZipBinary = + IOUtils.toByteArray( + getClass().getClassLoader().getResourceAsStream("simple_job_cancel.zip")); + + // Execute job upload with user + MvcResult jobUploadResult = + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(jobZipBinary) + .contentType(MediaType.APPLICATION_OCTET_STREAM)) + .andExpect(status().isOk()) + .andReturn(); + + DataJobVersion testDataJobVersion = + new ObjectMapper() + .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); + Assertions.assertNotNull(testDataJobVersion); + + String testJobVersionSha = testDataJobVersion.getVersionSha(); + Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); + + // Setup + String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); + + // Execute build and deploy job + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(dataJobDeploymentRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()) + .andReturn(); + + // manually start job execution + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\n" + + " \"args\": {\n" + + " \"key\": \"value\"\n" + + " },\n" + + " \"started_by\": \"schedule/runtime\"\n" + + "}")) + .andExpect(status().is(202)) + .andReturn(); + + // wait for pod to initialize + Thread.sleep(10000); + + // retrieve running job execution id. + var exc = + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + var gson = new Gson(); + ArrayList parsed = + gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); + String executionId = (String) parsed.get(0).get("id"); + + // cancel running execution + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, executionId)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } + + @Test + public void testJobCancellation_nonExistingJob() throws Exception { + + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, "executionId")) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java new file mode 100644 index 0000000000..48d9a6a4c8 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobEphemeralStorageIT.java @@ -0,0 +1,369 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import static org.awaitility.Awaitility.await; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.controlplane.model.data.DataJobExecution; +import com.vmware.taurus.controlplane.model.data.DataJobVersion; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MvcResult; +import java.util.concurrent.TimeUnit; +import java.util.List; +import com.fasterxml.jackson.core.type.TypeReference; +import java.util.stream.Collectors; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.UUID; + +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@Slf4j +@Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class}) +@TestPropertySource( + properties = { + "dataJob.readOnlyRootFileSystem=true", + }) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class DataJobEphemeralStorageIT extends BaseIT { + + private static final String TEST_JOB_NAME = + "ephemeral-storage-test-" + UUID.randomUUID().toString().substring(0, 8); + private static final Object DEPLOYMENT_ID = "testing-ephemeral-storage"; + private final ObjectMapper objectMapper = + new ObjectMapper() + .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; + + @AfterEach + public void cleanUp() throws Exception { + // delete job + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isOk()); + + // Execute delete deployment + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + } + + @BeforeEach + public void setup() throws Exception { + String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + // Execute create job + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()); + } + + @Test + public void testEphemeralStorageJob() throws Exception { + // Take the job zip as byte array + byte[] jobZipBinary = + IOUtils.toByteArray( + getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); + + // Execute job upload with user + MvcResult jobUploadResult = + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(jobZipBinary) + .contentType(MediaType.APPLICATION_OCTET_STREAM)) + .andExpect(status().isOk()) + .andReturn(); + + DataJobVersion testDataJobVersion = + new ObjectMapper() + .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); + Assertions.assertNotNull(testDataJobVersion); + + String testJobVersionSha = testDataJobVersion.getVersionSha(); + Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha)); + + // Setup + String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha); + + // Execute build and deploy job + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .content(dataJobDeploymentRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()) + .andReturn(); + + String opId = TEST_JOB_NAME + UUID.randomUUID().toString().toLowerCase(); + + // manually start job execution + mockMvc + .perform( + post(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .header(HEADER_X_OP_ID, opId) + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\n" + + " \"args\": {\n" + + " \"key\": \"value\"\n" + + " },\n" + + " \"started_by\": \"schedule/runtime\"\n" + + "}")) + .andExpect(status().is(202)) + .andReturn(); + + // wait for pod to initialize + Thread.sleep(10000); + + // retrieve running job execution id. + var exc = + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + var gson = new Gson(); + ArrayList parsed = + gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); + String executionId = (String) parsed.get(0).get("id"); + + // Check the data job execution status + checkDataJobExecutionStatus( + executionId, + DataJobExecution.StatusEnum.SUCCEEDED, + opId, + TEST_JOB_NAME, + TEST_TEAM_NAME, + "user"); + } + + private void checkDataJobExecutionStatus( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + try { + testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobDeploymentExecutionList( + executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionLogs(executionId, jobName, teamName, username); + } catch (Error e) { + try { + // print logs in case execution has failed + MvcResult dataJobExecutionLogsResult = + getExecuteLogs(executionId, jobName, teamName, username); + log.info( + "Job Execution {} logs:\n{}", + executionId, + dataJobExecutionLogsResult.getResponse().getContentAsString()); + } catch (Error ignore) { + } + throw e; + } + } + + private void testDataJobExecutionRead( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) { + + DataJobExecution[] dataJobExecution = new DataJobExecution[1]; + + await() + .atMost(5, TimeUnit.MINUTES) + .with() + .pollInterval(15, TimeUnit.SECONDS) + .until( + () -> { + String dataJobExecutionReadUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s", + teamName, jobName, executionId); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionReadUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + dataJobExecution[0] = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), + DataJobExecution.class); + if (dataJobExecution[0] == null) { + log.info("No response from server"); + } else { + log.info("Response from server " + dataJobExecution[0].getStatus()); + } + return dataJobExecution[0] != null + && executionStatus.equals(dataJobExecution[0].getStatus()); + }); + + assertDataJobExecutionValid(executionId, executionStatus, opId, dataJobExecution[0], jobName); + } + + private void testDataJobExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobExecutionListUrl = + String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); + Assertions.assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + Assertions.assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); + } + + private void testDataJobDeploymentExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobDeploymentExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + teamName, jobName, "release"); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobDeploymentExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); + Assertions.assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + Assertions.assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName); + } + + private void testDataJobExecutionLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); + Assertions.assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); + } + + @NotNull + private MvcResult getExecuteLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + String dataJobExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); + MvcResult dataJobExecutionLogsResult = + mockMvc + .perform(get(dataJobExecutionListUrl).with(user(username))) + .andExpect(status().isOk()) + .andReturn(); + return dataJobExecutionLogsResult; + } + + private void assertDataJobExecutionValid( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + DataJobExecution dataJobExecution, + String jobName) { + + Assertions.assertNotNull(dataJobExecution); + Assertions.assertEquals(executionId, dataJobExecution.getId()); + Assertions.assertEquals(jobName, dataJobExecution.getJobName()); + Assertions.assertEquals(executionStatus, dataJobExecution.getStatus()); + Assertions.assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); + Assertions.assertEquals(opId, dataJobExecution.getOpId()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java new file mode 100644 index 0000000000..4f8f34d5ac --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobGraphQLIT.java @@ -0,0 +1,293 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; + +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.NEW_TEST_TEAM_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_1; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_2; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_3; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_4; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_5; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_6; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class DataJobGraphQLIT extends BaseIT { + + private static final String DEFAULT_QUERY_WITH_VARS = + "query($filter: [Predicate], $search: String, $pageNumber: Int, $pageSize: Int) { " + + " jobs(pageNumber: $pageNumber, pageSize: $pageSize, filter: $filter, search: $search)" + + " { content { jobName config { team description " + + " schedule { scheduleCron } } } totalPages totalItems }" + + "}"; + + @Test + public void testGraphQLPagination() throws Exception { + createDummyJobs(); + + // Test listing of team jobs by filtering with team property + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param( + "variables", + "{" + + "\"filter\": [" + + " {" + + " \"property\": \"config.team\"," + + " \"pattern\": \"" + + TEST_TEAM_NAME + + "\"" + + " }" + + " ]," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (s.contains(TEST_JOB_1)) + && (s.contains(TEST_JOB_2)) + && (s.contains(TEST_JOB_6)) + && (!s.contains(TEST_JOB_3)) + && (!s.contains(TEST_JOB_4)) + && (!s.contains(TEST_JOB_5))))); + + // Test listing of team jobs by searching for team name + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param( + "variables", + "{" + + "\"search\": \"" + + NEW_TEST_TEAM_NAME + + "\"," + + "\"pageNumber\": 1," + + "\"pageSize\": 10" + + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (!s.contains(TEST_JOB_1)) + && (!s.contains(TEST_JOB_2)) + && (!s.contains(TEST_JOB_6)) + && (s.contains(TEST_JOB_3)) + && (s.contains(TEST_JOB_4)) + && (s.contains(TEST_JOB_5))))); + + // Test showing only first page + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param("variables", "{" + "\"pageNumber\": 1," + "\"pageSize\": 2" + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (s.contains(TEST_JOB_1)) + && (s.contains(TEST_JOB_2)) + && (!s.contains(TEST_JOB_3)) + && (!s.contains(TEST_JOB_4)) + && (!s.contains(TEST_JOB_5)) + && (!s.contains(TEST_JOB_6))))); + + // Test showing only middle page + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param("variables", "{" + "\"pageNumber\": 2," + "\"pageSize\": 2" + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (!s.contains(TEST_JOB_1)) + && (!s.contains(TEST_JOB_2)) + && (s.contains(TEST_JOB_3)) + && (s.contains(TEST_JOB_4)) + && (!s.contains(TEST_JOB_5)) + && (!s.contains(TEST_JOB_6))))); + + // Test showing only last page + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param("variables", "{" + "\"pageNumber\": 3," + "\"pageSize\": 2" + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (!s.contains(TEST_JOB_1)) + && (!s.contains(TEST_JOB_2)) + && (!s.contains(TEST_JOB_3)) + && (!s.contains(TEST_JOB_4)) + && (s.contains(TEST_JOB_5)) + && (s.contains(TEST_JOB_6))))); + + // Test showing only first page sorted DESC by jobName + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param( + "variables", + "{" + + "\"filter\": [" + + " {" + + " \"property\": \"jobName\"," + + " \"sort\": \"DESC\"" + + " }" + + " ]," + + "\"pageNumber\": 1," + + "\"pageSize\": 2" + + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (!s.contains(TEST_JOB_1)) + && (!s.contains(TEST_JOB_2)) + && (!s.contains(TEST_JOB_3)) + && (!s.contains(TEST_JOB_4)) + && (s.contains(TEST_JOB_5)) + && (s.contains(TEST_JOB_6))))); + + // Test showing custom filtered, sorted and paged data jobs by multiple parameters + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param("query", DEFAULT_QUERY_WITH_VARS) + .param( + "variables", + "{" + + "\"filter\": [" + + " {" + + " \"property\": \"config.team\"," + + " \"pattern\": \"" + + TEST_TEAM_NAME + + "\"" + + " }," + + " {" + + " \"property\": \"jobName\"," + + " \"sort\": \"DESC\"" + + " }" + + " ]," + + "\"pageNumber\": 1," + + "\"pageSize\": 2" + + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect( + content() + .string( + lambdaMatcher( + s -> + (!s.contains(TEST_JOB_1)) + && (s.contains(TEST_JOB_2)) + && (!s.contains(TEST_JOB_3)) + && (!s.contains(TEST_JOB_4)) + && (!s.contains(TEST_JOB_5)) + && (s.contains(TEST_JOB_6))))); + + deleteDummyJobs(); + } + + private void createDummyJobs() throws Exception { + // Setup by creating 3 jobs in 2 separate teams (6 jobs total) + String dataJobTestBodyOne = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_1); + createJob(dataJobTestBodyOne, TEST_TEAM_NAME); + + String dataJobTestBodyTwo = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_2); + createJob(dataJobTestBodyTwo, TEST_TEAM_NAME); + + String dataJobTestBodyThree = getDataJobRequestBody(NEW_TEST_TEAM_NAME, TEST_JOB_3); + createJob(dataJobTestBodyThree, NEW_TEST_TEAM_NAME); + + String dataJobTestBodyFour = getDataJobRequestBody(NEW_TEST_TEAM_NAME, TEST_JOB_4); + createJob(dataJobTestBodyFour, NEW_TEST_TEAM_NAME); + + String dataJobTestBodyFive = getDataJobRequestBody(NEW_TEST_TEAM_NAME, TEST_JOB_5); + createJob(dataJobTestBodyFive, NEW_TEST_TEAM_NAME); + + String dataJobTestBodySix = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_6); + createJob(dataJobTestBodySix, TEST_TEAM_NAME); + } + + private void deleteDummyJobs() throws Exception { + // Clean up + deleteJob(TEST_JOB_1, TEST_TEAM_NAME); + deleteJob(TEST_JOB_2, TEST_TEAM_NAME); + deleteJob(TEST_JOB_3, NEW_TEST_TEAM_NAME); + deleteJob(TEST_JOB_4, NEW_TEST_TEAM_NAME); + deleteJob(TEST_JOB_5, NEW_TEST_TEAM_NAME); + deleteJob(TEST_JOB_6, TEST_TEAM_NAME); + } + + private void createJob(String body, String teamName) throws Exception { + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", teamName)) + .with(user("user")) + .content(body) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()); + } + + private void deleteJob(String jobName, String teamName) throws Exception { + mockMvc + .perform( + delete(String.format("/data-jobs/for-team/%s/jobs/%s", teamName, jobName)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java new file mode 100644 index 0000000000..111ea9f663 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobPropertiesIT.java @@ -0,0 +1,88 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.properties.service.PropertiesRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +import java.util.HashMap; + +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class DataJobPropertiesIT extends BaseIT { + + @Autowired private PropertiesRepository propertiesRepository; + + @Test + public void testDataJobProperties() throws Exception { + // Setup + String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + // Execute create job (Post Create WebHook will return success for this call) + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()) + .andExpect( + header() + .string( + HttpHeaders.LOCATION, + lambdaMatcher( + s -> + s.endsWith( + String.format( + "/data-jobs/for-team/%s/jobs/%s", + TEST_TEAM_NAME, TEST_JOB_NAME))))); + + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/properties", + TEST_TEAM_NAME, TEST_JOB_NAME, "dev")) + .with(user("user"))) + .andExpect(status().isOk()) + .andExpect(content().string("{}")); + + var props = new HashMap(); + props.put("string_key", "string_value"); + props.put("int_key", 123); + props.put("bool_key", true); + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/properties", + TEST_TEAM_NAME, TEST_JOB_NAME, "dev")) + .with(user("user")) + .content(mapper.writeValueAsString(props)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isNoContent()); + + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/properties", + TEST_TEAM_NAME, TEST_JOB_NAME, "dev")) + .with(user("user"))) + .andExpect(status().isOk()) + .andExpect(content().json(mapper.writeValueAsString(props))); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java new file mode 100644 index 0000000000..11da9c0633 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobTerminationStatusIT.java @@ -0,0 +1,386 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.actuate.metrics.AutoConfigureMetrics; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; + +import com.vmware.taurus.controlplane.model.data.DataJobExecution; +import com.vmware.taurus.controlplane.model.data.DataJobExecutionRequest; +import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT; +import com.vmware.taurus.service.JobsRepository; + +@AutoConfigureMetrics +public class DataJobTerminationStatusIT extends BaseDataJobDeploymentIT { + + public static final String INFO_METRICS = "taurus_datajob_info"; + public static final String TERMINATION_STATUS_METRICS = "taurus_datajob_termination_status"; + public static final String HEADER_X_OP_ID = "X-OPID"; + + private static final Logger log = LoggerFactory.getLogger(DataJobTerminationStatusIT.class); + private final ObjectMapper objectMapper = + new ObjectMapper() + .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; + + @Autowired JobsRepository jobsRepository; + + // TODO split this test into job termination status test and data job execution test + /** + * This test aims to validate the data job execution and termination status monitoring logic in + * TPCS. For the purpose it does the following: + * + *

    + *
  • Creates a new data job (simple_job.zip), configured with: + *
      + *
    • schedule (e.g. */2 * * * *) + *
    • notification emails + *
    • enable_execution_notifications set to false + *
    + *
  • Deploys the data job + *
  • Wait for the deployment to complete + *
  • Executes the data job + *
  • Checks data job execution status + *
  • Wait until a single data job execution to complete (using the awaitility for this) + *
  • Checks data job execution status + *
  • Make a rest call to the (/data-jobs/debug/prometheus) + *
  • Parse the result and validate that: + *
      + *
    • there is a taurus_datajob_info metrics for the data job used for testing and that + * it has the appropriate email labels (they should be empty because + * enable_execution_notifications is false) + *
    • there is a taurus_datajob_termination_status metrics for the data job used for + * testing and it has the appropriate value (0.0 in this case; i.e. the job should + * have succeeded) + *
    + *
  • Finally, delete the job deployment and the leftover K8s jobs. + *
+ */ + @Test + public void testDataJobTerminationStatus(String jobName, String teamName, String username) + throws Exception { + // Execute data job + String opId = jobName + UUID.randomUUID().toString().toLowerCase(); + DataJobExecutionRequest dataJobExecutionRequest = + new DataJobExecutionRequest().startedBy(username); + + String triggerDataJobExecutionUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + teamName, jobName, "release"); + MvcResult dataJobExecutionResponse = + mockMvc + .perform( + post(triggerDataJobExecutionUrl) + .with(user(username)) + .header(HEADER_X_OP_ID, opId) + .content(mapper.writeValueAsString(dataJobExecutionRequest)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().is(202)) + .andReturn(); + + // Check the data job execution status + String location = dataJobExecutionResponse.getResponse().getHeader("location"); + String executionId = location.substring(location.lastIndexOf("/") + 1); + checkDataJobExecutionStatus( + executionId, DataJobExecution.StatusEnum.SUCCEEDED, opId, jobName, teamName, username); + + // Wait for the job execution to complete, polling every 5 seconds + // See: https://github.com/awaitility/awaitility/wiki/Usage + await() + .atMost(10, TimeUnit.MINUTES) + .with() + .pollInterval(15, TimeUnit.SECONDS) + .until(terminationMetricsAreAvailable()); + + String scrape = scrapeMetrics(); + + // Validate that all metrics for the executed data job are correctly exposed + // First, validate that there is a taurus_datajob_info metrics for the data job + var match = findMetricsWithLabel(scrape, INFO_METRICS, "data_job", jobName); + assertTrue( + match.isPresent(), + String.format("Could not find %s metrics for the data job %s", INFO_METRICS, jobName)); + + // Validate the labels of the taurus_datajob_info metrics + String line = match.get(); + assertLabelEquals(INFO_METRICS, teamName, "team", line); + assertLabelEquals(INFO_METRICS, "", "email_notified_on_success", line); + assertLabelEquals(INFO_METRICS, "", "email_notified_on_platform_error", line); + assertLabelEquals(INFO_METRICS, "", "email_notified_on_user_error", line); + + // Validate that there is a taurus_datajob_info metrics for the data job + match = findMetricsWithLabel(scrape, TERMINATION_STATUS_METRICS, "data_job", jobName); + assertTrue( + match.isPresent(), + String.format( + "Could not find %s metrics for the data job %s", TERMINATION_STATUS_METRICS, jobName)); + + // Validate that the metrics has a value of 0.0 (i.e. Success) + System.out.println(match.get().trim()); + assertTrue( + match.get().trim().endsWith("0.0"), + "The value of the taurus_datajob_termination_status metrics does not match. It was actually" + + " " + + match.get()); + + // Check the data job execution status + checkDataJobExecutionStatus( + executionId, DataJobExecution.StatusEnum.SUCCEEDED, opId, jobName, teamName, username); + } + + private String scrapeMetrics() throws Exception { + return mockMvc + .perform(get("/data-jobs/debug/prometheus")) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + } + + private Callable terminationMetricsAreAvailable() { + return () -> scrapeMetrics().contains(TERMINATION_STATUS_METRICS); + } + + private static Optional findMetricsWithLabel( + CharSequence input, String metrics, String label, String labelValue) { + Pattern pattern = + Pattern.compile( + String.format("%s\\{.*%s=\"%s\"[^\\}]*\\} (.*)", metrics, label, labelValue), + Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(input); + if (!matcher.find()) { + return Optional.empty(); + } + return Optional.of(matcher.group()); + } + + private static void assertLabelEquals( + String metrics, String expectedValue, String label, String line) { + // Label value is captured in the first regex group + Matcher matcher = + Pattern.compile(String.format(".*%s=\"([^\"]*)\".*", label), Pattern.CASE_INSENSITIVE) + .matcher(line); + assertTrue( + matcher.find(), + String.format("The metrics %s does not have a matching label %s", metrics, label)); + String actualValue = matcher.group(1); + assertEquals( + expectedValue, + actualValue, + String.format( + "The metrics %s does not have correct value for label %s. Expected: %s, Actual: %s", + metrics, label, expectedValue, actualValue)); + } + + private void checkDataJobExecutionStatus( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + try { + testDataJobExecutionRead(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionList(executionId, executionStatus, opId, jobName, teamName, username); + testDataJobDeploymentExecutionList( + executionId, executionStatus, opId, jobName, teamName, username); + testDataJobExecutionLogs(executionId, jobName, teamName, username); + } catch (Error e) { + try { + // print logs in case execution has failed + MvcResult dataJobExecutionLogsResult = + getExecuteLogs(executionId, jobName, teamName, username); + log.info( + "Job Execution {} logs:\n{}", + executionId, + dataJobExecutionLogsResult.getResponse().getContentAsString()); + } catch (Error ignore) { + } + throw e; + } + } + + private void testDataJobExecutionRead( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) { + + DataJobExecution[] dataJobExecution = new DataJobExecution[1]; + + await() + .atMost(5, TimeUnit.MINUTES) + .with() + .pollInterval(15, TimeUnit.SECONDS) + .until( + () -> { + String dataJobExecutionReadUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s", + teamName, jobName, executionId); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionReadUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + dataJobExecution[0] = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), + DataJobExecution.class); + if (dataJobExecution[0] == null) { + log.info("No response from server"); + } else { + log.info("Response from server " + dataJobExecution[0].getStatus()); + } + return dataJobExecution[0] != null + && executionStatus.equals(dataJobExecution[0].getStatus()); + }); + + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecution[0], jobName, username); + } + + private void testDataJobExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobExecutionListUrl = + String.format("/data-jobs/for-team/%s/jobs/%s/executions", teamName, jobName); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); + assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName, username); + } + + private void testDataJobDeploymentExecutionList( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + String jobName, + String teamName, + String username) + throws Exception { + + String dataJobDeploymentExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + teamName, jobName, "release"); + MvcResult dataJobExecutionResult = + mockMvc + .perform( + get(dataJobDeploymentExecutionListUrl) + .with(user(username)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andReturn(); + + List dataJobExecutions = + objectMapper.readValue( + dataJobExecutionResult.getResponse().getContentAsString(), new TypeReference<>() {}); + assertNotNull(dataJobExecutions); + dataJobExecutions = + dataJobExecutions.stream() + .filter(e -> e.getId().equals(executionId)) + .collect(Collectors.toList()); + assertEquals(1, dataJobExecutions.size()); + assertDataJobExecutionValid( + executionId, executionStatus, opId, dataJobExecutions.get(0), jobName, username); + } + + private void testDataJobExecutionLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + MvcResult dataJobExecutionLogsResult = getExecuteLogs(executionId, jobName, teamName, username); + assertFalse(dataJobExecutionLogsResult.getResponse().getContentAsString().isEmpty()); + } + + @NotNull + private MvcResult getExecuteLogs( + String executionId, String jobName, String teamName, String username) throws Exception { + String dataJobExecutionListUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/executions/%s/logs", teamName, jobName, executionId); + MvcResult dataJobExecutionLogsResult = + mockMvc + .perform(get(dataJobExecutionListUrl).with(user(username))) + .andExpect(status().isOk()) + .andReturn(); + return dataJobExecutionLogsResult; + } + + private void assertDataJobExecutionValid( + String executionId, + DataJobExecution.StatusEnum executionStatus, + String opId, + DataJobExecution dataJobExecution, + String jobName, + String username) { + + assertNotNull(dataJobExecution); + assertEquals(executionId, dataJobExecution.getId()); + assertEquals(jobName, dataJobExecution.getJobName()); + assertEquals(executionStatus, dataJobExecution.getStatus()); + assertEquals(DataJobExecution.TypeEnum.MANUAL, dataJobExecution.getType()); + assertEquals(username + "/" + "user", dataJobExecution.getStartedBy()); + assertEquals(opId, dataJobExecution.getOpId()); + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java new file mode 100644 index 0000000000..5527cad0cd --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseDataJobDeploymentIT.java @@ -0,0 +1,88 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it.common; + +import static org.awaitility.Awaitility.await; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.controlplane.model.data.DataJobExecutionRequest; + +/** + * It combines all necessary annotations and constants for tests that need an already deployed data + * job. + * + *

The test just needs to extend this class, and it will have access to the already deployed data + * job. + */ +@Import({BaseDataJobDeploymentIT.TaskExecutorConfig.class}) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +@ExtendWith(DataJobDeploymentExtension.class) +public abstract class BaseDataJobDeploymentIT extends BaseIT { + + @TestConfiguration + public static class TaskExecutorConfig { + @Bean + @Primary + public TaskExecutor taskExecutor() { + // Deployment methods are non-blocking (Async) which makes them harder to test. + // Making them sync for the purposes of this test. + return new SyncTaskExecutor(); + } + } + + protected MvcResult executeDataJob( + String jobName, String teamName, String username, String opId) { + // Execute data job + if (opId == null) { + opId = jobName + UUID.randomUUID().toString().toLowerCase(); + } + + DataJobExecutionRequest dataJobExecutionRequest = + new DataJobExecutionRequest().startedBy(username); + + String triggerDataJobExecutionUrl = + String.format( + "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions", + teamName, jobName, "release"); + + // Wait for the job execution to complete, polling every 15 seconds + // See: https://github.com/awaitility/awaitility/wiki/Usage + String finalOpId = opId; + return await() + .atMost(5, TimeUnit.MINUTES) + .with() + .pollInterval(15, TimeUnit.SECONDS) + .until( + () -> + mockMvc + .perform( + post(triggerDataJobExecutionUrl) + .with(user(username)) + .header(HEADER_X_OP_ID, finalOpId) + .content(mapper.writeValueAsString(dataJobExecutionRequest)) + .contentType(MediaType.APPLICATION_JSON)) + .andReturn(), + mvcResult -> mvcResult.getResponse().getStatus() == 202); + } +} From 2789200e6387fcab46b9152ebe67bc27834293c1 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 16:38:03 +0000 Subject: [PATCH 31/69] vdk-control-service: set registry name correctly. Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 64d1d616c9..701bf17a01 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -48,8 +48,10 @@ control_service_build_image: - apk --no-cache add git openjdk11-jdk --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community - export TAG=$(git rev-parse --short HEAD) - cd projects/control-service/projects - - ./gradlew -p ./model build publishToMavenLocal - - ./gradlew build jacocoTestReport -x integrationTest + - ./gradlew -p ./model build publishToMavenLocal --info --stacktrace + - ./gradlew build jacocoTestReport -x integrationTest --info --stacktrace + - ./gradlew :pipelines_control_service:docker --info --stacktrace -Pversion=$TAG + retry: !reference [.control_service_retry, retry_options] coverage: "/ - Line Coverage: ([0-9.]+)%/" artifacts: when: always @@ -64,8 +66,6 @@ control_service_build_image: - main changes: *control_service_change_locations - - control_service_integration_test: extends: .control_service_base_build stage: build From 3e59c3e2b99cbc9c234acda8d1f719f8e753b23c Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 16:40:11 +0000 Subject: [PATCH 32/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../datajobs/it/common/JobExecutionUtil.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java new file mode 100644 index 0000000000..9d750cccf8 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/JobExecutionUtil.java @@ -0,0 +1,49 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it.common; + +import java.time.OffsetDateTime; + +import com.vmware.taurus.service.JobExecutionRepository; +import com.vmware.taurus.service.model.DataJob; +import com.vmware.taurus.service.model.DataJobExecution; +import com.vmware.taurus.service.model.ExecutionStatus; +import com.vmware.taurus.service.model.ExecutionType; + +public class JobExecutionUtil { + + public static DataJobExecution createDataJobExecution( + JobExecutionRepository jobExecutionRepository, + String executionId, + DataJob dataJob, + OffsetDateTime startTime, + OffsetDateTime endTime, + ExecutionStatus executionStatus) { + + var jobExecution = + DataJobExecution.builder() + .id(executionId) + .dataJob(dataJob) + .startTime(startTime) + .endTime(endTime) + .type(ExecutionType.MANUAL) + .status(executionStatus) + .resourcesCpuRequest(1F) + .resourcesCpuLimit(2F) + .resourcesMemoryRequest(500) + .resourcesMemoryLimit(1000) + .message("message") + .lastDeployedBy("test_user") + .lastDeployedDate(OffsetDateTime.now()) + .jobVersion("test_version") + .jobSchedule("*/5 * * * *") + .opId("test_op_id") + .vdkVersion("test_vdk_version") + .build(); + + return jobExecutionRepository.save(jobExecution); + } +} From 9bc66e0feafd638d879d085357cfad6af9de9aa2 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:21:36 +0000 Subject: [PATCH 33/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 9437913e45..177f0637ad 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -57,9 +57,6 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { private static final String TEST_JOB_NAME = "private-docker-builder-test-" + UUID.randomUUID().toString().substring(0, 8); private static final Object DEPLOYMENT_ID = "private-docker-builder"; - private final ObjectMapper objectMapper = - new ObjectMapper() - .registerModule(new JavaTimeModule()); // Used for converting to OffsetDateTime; @Value("${datajobs.builder.registrySecret.content.testOnly:}") private String dataJobsBuilderRegistrySecretContent; From 29c87fc4c006bc95e04e390e12b63a0b210f22ab Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 18:22:11 +0000 Subject: [PATCH 34/69] Google Java Format --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 177f0637ad..111c499f8d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.gson.Gson; import com.google.gson.internal.LinkedTreeMap; import com.vmware.taurus.ControlplaneApplication; From de336a57006e7c2df9858413aec078b99b0fc19b Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:22:37 +0000 Subject: [PATCH 35/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/datajobs/it/DataJobCrudIT.java | 460 ++++++++++++++++++ 1 file changed, 460 insertions(+) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java new file mode 100644 index 0000000000..5d403dac32 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java @@ -0,0 +1,460 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.datajobs.it; + +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.service.JobsRepository; +import com.vmware.taurus.service.credentials.JobCredentialsService; +import com.vmware.taurus.service.model.DataJob; +import org.apache.commons.lang3.ArrayUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.NEW_TEST_TEAM_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_CLIENT_ERROR_JOB_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_CLIENT_ERROR_TEAM; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_JOB_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_RETRIED_JOB_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_RETRIED_TEAM; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_INTERNAL_ERROR_TEAM; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_JOB_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME; +import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_WRONG_NAME; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = ControlplaneApplication.class) +public class DataJobCrudIT extends BaseIT { + + @Autowired private JobsRepository jobsRepository; + + @Test + public void testDataJobCrud() throws Exception { + // Setup + String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + // Execute create job (Post Create WebHook will return success for this call) + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()) + .andExpect( + header() + .string( + HttpHeaders.LOCATION, + lambdaMatcher( + s -> + s.endsWith( + String.format( + "/data-jobs/for-team/%s/jobs/%s", + TEST_TEAM_NAME, TEST_JOB_NAME))))); + // Validate - the job is created + Assertions.assertTrue(jobsRepository.existsById(TEST_JOB_NAME)); + + // Execute create job with no user + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isUnauthorized()); + + testDataJobPostCreateWebHooks(); + + // Execute get swagger with no user + mockMvc + .perform( + get("/data-jobs/swagger-ui.html") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mockMvc + .perform( + get("/data-jobs/webjars/springfox-swagger-ui/swagger-ui.css.map") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mockMvc + .perform( + get("/data-jobs/webjars/springfox-swagger-ui/swagger-ui-bundle.js.map") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mockMvc + .perform( + get("/data-jobs/swagger-resources/configuration/ui") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mockMvc + .perform( + get("/data-jobs/swagger-resources/configuration/security") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mockMvc + .perform( + get("/data-jobs/swagger-resources") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mockMvc + .perform( + get("/data-jobs/v2/api-docs") + .content(dataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + // Validation + // Validate keytab created + var keytabSecretName = JobCredentialsService.getJobKeytabKubernetesSecretName(TEST_JOB_NAME); + var keytabSecretData = dataJobsKubernetesService.getSecretData(keytabSecretName); + Assertions.assertFalse(keytabSecretData.isEmpty()); + Assertions.assertTrue(keytabSecretData.containsKey("keytab")); + Assertions.assertTrue(ArrayUtils.isNotEmpty(keytabSecretData.get("keytab"))); + + // Validate persisted job + var jobFromDbOptional = jobsRepository.findById(TEST_JOB_NAME); + Assertions.assertTrue(jobFromDbOptional.isPresent()); + var jobFromDb = jobFromDbOptional.get(); + Assertions.assertEquals(TEST_JOB_SCHEDULE, jobFromDb.getJobConfig().getSchedule()); + + // Execute list jobs + // deprecated jobsList in favour of jobsQuery + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .param( + "query", + "query($filter: [Predicate], $pageNumber: Int) {" + + " jobs(pageNumber: $pageNumber, filter: $filter) {" + + " content {" + + " jobName" + + " }" + + " }" + + "}") + .param( + "variables", + "{" + + "\"filter\": [" + + " {" + + " \"property\": \"config.team\"," + + " \"pattern\": \"" + + TEST_TEAM_NAME + + "\"" + + " }" + + " ]," + + "\"pageNumber\": 1" + + "}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(lambdaMatcher(s -> (s.contains(TEST_JOB_NAME))))); + + // Execute list jobs with no user + // deprecated jobsList in favour of jobsQuery + mockMvc + .perform(get(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME))) + .andExpect(status().isUnauthorized()) + .andExpect(content().string(lambdaMatcher(s -> s.isBlank()))); + + // Execute get job with user + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isOk()) + .andExpect(content().string(lambdaMatcher(s -> s.contains(TEST_JOB_NAME)))) + .andExpect(content().string(lambdaMatcher(s -> s.contains(TEST_JOB_SCHEDULE)))); + + // Execute get job with user and wrong team + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_WRONG_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Execute get job without user + mockMvc + .perform( + get(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME))) + .andExpect(status().isUnauthorized()) + .andExpect(content().string(lambdaMatcher(s -> s.isBlank()))); + + // Execute update job team with user + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s/team/%s", + TEST_TEAM_WRONG_NAME, TEST_JOB_NAME, NEW_TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Execute update job team with no user + mockMvc + .perform( + put( + String.format( + "/data-jobs/for-team/%s/jobs/%s/team/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, NEW_TEST_TEAM_NAME))) + .andExpect(status().isUnauthorized()); + + // Execute update job team with same team and user + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s/team/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isOk()); + + // Execute update job team with empty team + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s/team/ ", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Execute update job team with user + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s/team/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isOk()); + + // Execute update job team with non existing job + mockMvc + .perform( + put(String.format("/data-jobs/for-team/%s/jobs/missing-job/team/", TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Execute update job team with empty job name + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/ /team/%s", TEST_TEAM_NAME, NEW_TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Execute update job team with missing job name + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs//team/%s", TEST_TEAM_NAME, NEW_TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Execute get not allowed method to job team endpoint + mockMvc + .perform( + get(String.format( + "/data-jobs/for-team/%s/jobs/%s/team/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isMethodNotAllowed()); + + // Execute put method with user and with wrong team + // TODO: uncomment and extend tests to cover all put operations not only NotFound + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_WRONG_NAME, TEST_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON) + .content(dataJobRequestBody)) + .andExpect(status().isNotFound()); + + // Execute delete job with no user + mockMvc + .perform( + delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isUnauthorized()); + + // Execute delete job with user and wrong team + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_WRONG_NAME, TEST_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()); + + // Execute delete job with user + mockMvc + .perform( + delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + // Execute update job team after job is deleted from db and is missing + mockMvc + .perform( + put(String.format( + "/data-jobs/for-team/%s/jobs/%s/team/%s", + TEST_TEAM_NAME, TEST_JOB_NAME, TEST_TEAM_NAME)) + .with(user("user"))) + .andExpect(status().isNotFound()); + + // Validate keytab deleted + keytabSecretData = dataJobsKubernetesService.getSecretData(keytabSecretName); + Assertions.assertTrue(keytabSecretData.isEmpty()); + + // Validate job deleted from db + Assertions.assertFalse(jobsRepository.existsById(TEST_JOB_NAME)); + + testDataJobPostDeleteWebHooks(); + } + + private void testDataJobPostCreateWebHooks() throws Exception { + // Post Create WebHook will prevent job creation and the result will be propagated error code + // with no job created + String clientErrorDataJobRequestBody = + getDataJobRequestBody(TEST_CLIENT_ERROR_TEAM, TEST_CLIENT_ERROR_JOB_NAME); + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_CLIENT_ERROR_TEAM)) + .with(user("user")) + .content(clientErrorDataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + // Validate - the job is NOT created + Assertions.assertFalse(jobsRepository.existsById(TEST_CLIENT_ERROR_JOB_NAME)); + + // Post Create WebHook will prevent job creation and the result will be error 503 with no job + // created + String internalServerErrorDataJobRequestBody = + getDataJobRequestBody(TEST_INTERNAL_ERROR_TEAM, TEST_INTERNAL_ERROR_JOB_NAME); + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_INTERNAL_ERROR_TEAM)) + .with(user("user")) + .content(internalServerErrorDataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isServiceUnavailable()); + // Validate - the job is NOT created + Assertions.assertFalse(jobsRepository.existsById(TEST_INTERNAL_ERROR_JOB_NAME)); + + // Post Create WebHook will retry 2 times and finally will allow job creation + String retriedErrorDataJobRequestBody = + getDataJobRequestBody( + TEST_INTERNAL_ERROR_RETRIED_TEAM, TEST_INTERNAL_ERROR_RETRIED_JOB_NAME); + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_INTERNAL_ERROR_RETRIED_TEAM)) + .with(user("user")) + .content(retriedErrorDataJobRequestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()); + // Validate - the job is created + Assertions.assertTrue(jobsRepository.existsById(TEST_INTERNAL_ERROR_RETRIED_JOB_NAME)); + } + + private void testDataJobPostDeleteWebHooks() throws Exception { + // Add test job to the repository + DataJob clientErrorEntity = + getDataJobRepositoryModel(TEST_CLIENT_ERROR_TEAM, TEST_CLIENT_ERROR_JOB_NAME); + jobsRepository.save(clientErrorEntity); + // Post Delete WebHook will prevent job deletion and the result will be propagated error code + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s", + TEST_CLIENT_ERROR_TEAM, TEST_CLIENT_ERROR_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + // Validate - the job is NOT deleted + Assertions.assertTrue(jobsRepository.existsById(TEST_CLIENT_ERROR_JOB_NAME)); + // Clean Up + jobsRepository.delete(clientErrorEntity); + + // Add test job to the repository + DataJob internalServerEntity = + getDataJobRepositoryModel(TEST_INTERNAL_ERROR_TEAM, TEST_INTERNAL_ERROR_JOB_NAME); + jobsRepository.save(internalServerEntity); + // Post Delete WebHook will prevent job deletion and the result will be error 503 + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s", + TEST_INTERNAL_ERROR_TEAM, TEST_INTERNAL_ERROR_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isServiceUnavailable()); + // Validate - the job is NOT deleted + Assertions.assertTrue(jobsRepository.existsById(TEST_INTERNAL_ERROR_JOB_NAME)); + // Clean Up + jobsRepository.delete(internalServerEntity); + + // Add test job to the repository + DataJob internalServerRetriedEntity = + getDataJobRepositoryModel( + TEST_INTERNAL_ERROR_RETRIED_TEAM, TEST_INTERNAL_ERROR_RETRIED_JOB_NAME); + jobsRepository.save(internalServerRetriedEntity); + // Post Create WebHook will retry 2 times and finally will allow job creation + mockMvc + .perform( + delete( + String.format( + "/data-jobs/for-team/%s/jobs/%s", + TEST_INTERNAL_ERROR_RETRIED_TEAM, TEST_INTERNAL_ERROR_RETRIED_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + // Validate - the job is deleted + Assertions.assertFalse(jobsRepository.existsById(TEST_INTERNAL_ERROR_RETRIED_JOB_NAME)); + // Clean Up + jobsRepository.delete(internalServerEntity); + } + + @Test + public void testDataJobCreateDeleteIdempotency() throws Exception { + String dataJobTestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME); + + mockMvc + .perform( + post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME)) + .with(user("user")) + .content(dataJobTestBody) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()); + + mockMvc + .perform( + delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + mockMvc + .perform( + delete(String.format("/data-jobs/for-team/%s/jobs/%s", TEST_TEAM_NAME, TEST_JOB_NAME)) + .with(user("user")) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()); + } +} From 6adfd333d5b30759671064bb38d0a9435007b214 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:36:04 +0000 Subject: [PATCH 36/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../it/PrivateBuilderDockerRepoIT.java | 16 +++++------- .../it/common/DockerConfigJsonUtils.java | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 111c499f8d..5949024d15 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -7,11 +7,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.gson.Gson; import com.google.gson.internal.LinkedTreeMap; import com.vmware.taurus.ControlplaneApplication; import com.vmware.taurus.controlplane.model.data.DataJobVersion; import com.vmware.taurus.datajobs.it.common.BaseIT; +import com.vmware.taurus.datajobs.it.common.DockerConfigJsonUtils; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1SecretBuilder; @@ -53,6 +55,7 @@ classes = ControlplaneApplication.class) public class PrivateBuilderDockerRepoIT extends BaseIT { + private static final String TEST_JOB_NAME = "private-docker-builder-test-" + UUID.randomUUID().toString().substring(0, 8); private static final Object DEPLOYMENT_ID = "private-docker-builder"; @@ -61,7 +64,7 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { private String dataJobsBuilderRegistrySecretContent; public void createBuilderImagePullSecret(String namespaceName) - throws ApiException, JsonProcessingException { + throws Exception { new CoreV1Api(controlKubernetesService.getClient()) .createNamespacedSecret( namespaceName, @@ -71,15 +74,8 @@ public void createBuilderImagePullSecret(String namespaceName) .withNamespace(namespaceName) .endMetadata() .withStringData( - Map.of( - ".dockerconfigjson", - new ObjectMapper() - .writeValueAsString( - Map.of( - "auths", - Map.of( - "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - Map.of("auth", dataJobsBuilderRegistrySecretContent)))))) + DockerConfigJsonUtils.create("vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + dataJobsBuilderRegistrySecretContent)) .withType("kubernetes.io/dockerconfigjson") .build(), null, diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java new file mode 100644 index 0000000000..f20575f9a2 --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java @@ -0,0 +1,25 @@ +package com.vmware.taurus.datajobs.it.common; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.Map; + +/** + * Responsible for creating k8s json secret format given a secret and also a repo. + */ +public class DockerConfigJsonUtils { + private static final ObjectMapper objectMapper = new ObjectMapper(); + + + public static Map create(String repo, String secret) throws JsonProcessingException { + return Map.of( + ".dockerconfigjson", + objectMapper.writeValueAsString( + Map.of("auths", + Map.of( + repo, + Map.of("auth", secret))))); + } + +} From 37a0d632b0925195a537b1e04a9febe402a93f9a Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 18:36:33 +0000 Subject: [PATCH 37/69] Google Java Format --- .../it/PrivateBuilderDockerRepoIT.java | 13 ++++------- .../it/common/DockerConfigJsonUtils.java | 23 +++++++------------ 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 5949024d15..00e808b768 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -5,16 +5,13 @@ package com.vmware.taurus.datajobs.it; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.gson.Gson; import com.google.gson.internal.LinkedTreeMap; import com.vmware.taurus.ControlplaneApplication; import com.vmware.taurus.controlplane.model.data.DataJobVersion; import com.vmware.taurus.datajobs.it.common.BaseIT; import com.vmware.taurus.datajobs.it.common.DockerConfigJsonUtils; -import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1SecretBuilder; import lombok.extern.slf4j.Slf4j; @@ -34,7 +31,6 @@ import org.springframework.test.web.servlet.MvcResult; import java.util.ArrayList; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -55,7 +51,6 @@ classes = ControlplaneApplication.class) public class PrivateBuilderDockerRepoIT extends BaseIT { - private static final String TEST_JOB_NAME = "private-docker-builder-test-" + UUID.randomUUID().toString().substring(0, 8); private static final Object DEPLOYMENT_ID = "private-docker-builder"; @@ -63,8 +58,7 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { @Value("${datajobs.builder.registrySecret.content.testOnly:}") private String dataJobsBuilderRegistrySecretContent; - public void createBuilderImagePullSecret(String namespaceName) - throws Exception { + public void createBuilderImagePullSecret(String namespaceName) throws Exception { new CoreV1Api(controlKubernetesService.getClient()) .createNamespacedSecret( namespaceName, @@ -74,8 +68,9 @@ public void createBuilderImagePullSecret(String namespaceName) .withNamespace(namespaceName) .endMetadata() .withStringData( - DockerConfigJsonUtils.create("vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - dataJobsBuilderRegistrySecretContent)) + DockerConfigJsonUtils.create( + "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + dataJobsBuilderRegistrySecretContent)) .withType("kubernetes.io/dockerconfigjson") .build(), null, diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java index f20575f9a2..31ded7a0b3 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java @@ -5,21 +5,14 @@ import java.util.Map; -/** - * Responsible for creating k8s json secret format given a secret and also a repo. - */ +/** Responsible for creating k8s json secret format given a secret and also a repo. */ public class DockerConfigJsonUtils { - private static final ObjectMapper objectMapper = new ObjectMapper(); - - - public static Map create(String repo, String secret) throws JsonProcessingException { - return Map.of( - ".dockerconfigjson", - objectMapper.writeValueAsString( - Map.of("auths", - Map.of( - repo, - Map.of("auth", secret))))); - } + private static final ObjectMapper objectMapper = new ObjectMapper(); + public static Map create(String repo, String secret) + throws JsonProcessingException { + return Map.of( + ".dockerconfigjson", + objectMapper.writeValueAsString(Map.of("auths", Map.of(repo, Map.of("auth", secret))))); + } } From 9ac12f0c76ddc01c82846bef4d703b6b0ad6fcd8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 18:38:22 +0000 Subject: [PATCH 38/69] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../taurus/datajobs/it/common/DockerConfigJsonUtils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java index 31ded7a0b3..b7324f1903 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java @@ -1,3 +1,8 @@ +/* + * Copyright 2021 VMware, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package com.vmware.taurus.datajobs.it.common; import com.fasterxml.jackson.core.JsonProcessingException; From c49f4f0147a628245cff29737d65ffcee6b916d2 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:44:15 +0000 Subject: [PATCH 39/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/service/deploy/JobImageBuilder.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index 1c916a78c8..c909dc1442 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -106,6 +106,17 @@ public JobImageBuilder( this.kubernetesResources = kubernetesResources; } + /** + * see {@link JobImageBuilder#buildImage(String, DataJob, JobDeployment, Boolean, String)}} + */ + public boolean buildImage( + String imageName, + DataJob dataJob, + JobDeployment jobDeployment, + Boolean sendNotification) + throws ApiException, IOException, InterruptedException { + return buildImage(imageName,dataJob , jobDeployment, sendNotification, null); + } /** * Builds and pushes a docker image for a data job. Runs a job on k8s which is responsible for * building and pushing the data job image. This call will block until the builder job has @@ -115,6 +126,7 @@ public JobImageBuilder( * @param dataJob Information about the data job. * @param jobDeployment Information about the data job deployment. * @param sendNotification + * @param registrySecret should be the name of the secret that exists in the kubernetes cluster/namespace already. * @return True if build and push was successful. False otherwise. * @throws ApiException * @throws IOException From 8a393a92fd1f657db76dd26996440674d2e371c9 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:45:43 +0000 Subject: [PATCH 40/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../vmware/taurus/service/deploy/JobImageBuilderTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java index e1741d316f..8a9929d295 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java @@ -78,7 +78,7 @@ public void buildImage_notExist_success() throws InterruptedException, ApiExcept jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true, null); + var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true); verify(kubernetesService) .createJob( @@ -120,7 +120,7 @@ public void buildImage_builderRunning_oldBuilderDeleted() jobDeployment.setEnabled(true); var result = - jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true, null); + jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService, times(2)).deleteJob(TEST_BUILDER_IMAGE_NAME); verify(kubernetesService) @@ -155,7 +155,7 @@ public void buildImage_imageExists_buildSkipped() jobDeployment.setEnabled(true); var result = - jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true, null); + jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService, never()) .createJob( @@ -195,7 +195,7 @@ public void buildImage_jobFailed_failure() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true, null); + var result = jobImageBuilder.buildImage("test-image", testDataJob, jobDeployment, true); verify(kubernetesService) .createJob( From a148478fd59906e8b5bd711dc0dc6728a46636cd Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 18:46:17 +0000 Subject: [PATCH 41/69] Google Java Format --- .../taurus/service/deploy/JobImageBuilder.java | 16 ++++++---------- .../service/deploy/JobImageBuilderTest.java | 6 ++---- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index c909dc1442..c7adc10e5e 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -106,16 +106,11 @@ public JobImageBuilder( this.kubernetesResources = kubernetesResources; } - /** - * see {@link JobImageBuilder#buildImage(String, DataJob, JobDeployment, Boolean, String)}} - */ + /** see {@link JobImageBuilder#buildImage(String, DataJob, JobDeployment, Boolean, String)}} */ public boolean buildImage( - String imageName, - DataJob dataJob, - JobDeployment jobDeployment, - Boolean sendNotification) - throws ApiException, IOException, InterruptedException { - return buildImage(imageName,dataJob , jobDeployment, sendNotification, null); + String imageName, DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification) + throws ApiException, IOException, InterruptedException { + return buildImage(imageName, dataJob, jobDeployment, sendNotification, null); } /** * Builds and pushes a docker image for a data job. Runs a job on k8s which is responsible for @@ -126,7 +121,8 @@ public boolean buildImage( * @param dataJob Information about the data job. * @param jobDeployment Information about the data job deployment. * @param sendNotification - * @param registrySecret should be the name of the secret that exists in the kubernetes cluster/namespace already. + * @param registrySecret should be the name of the secret that exists in the kubernetes + * cluster/namespace already. * @return True if build and push was successful. False otherwise. * @throws ApiException * @throws IOException diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java index 8a9929d295..ab9aa90a55 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/JobImageBuilderTest.java @@ -119,8 +119,7 @@ public void buildImage_builderRunning_oldBuilderDeleted() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = - jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); + var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService, times(2)).deleteJob(TEST_BUILDER_IMAGE_NAME); verify(kubernetesService) @@ -154,8 +153,7 @@ public void buildImage_imageExists_buildSkipped() jobDeployment.setGitCommitSha("test-commit"); jobDeployment.setEnabled(true); - var result = - jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); + var result = jobImageBuilder.buildImage(TEST_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService, never()) .createJob( From 57a2314441f20449222312fe1d28b38e8502b885 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:49:16 +0000 Subject: [PATCH 42/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/datajobs/it/common/DockerConfigJsonUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java index b7324f1903..e9b37a9e76 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java @@ -10,7 +10,8 @@ import java.util.Map; -/** Responsible for creating k8s json secret format given a secret and also a repo. */ +/** Responsible for creating k8s formatted docker secret. Parameters are secret and the repo. */ + public class DockerConfigJsonUtils { private static final ObjectMapper objectMapper = new ObjectMapper(); From d192218ce91ed417f14afdb3061a8e3633f3a8bf Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 18:50:08 +0000 Subject: [PATCH 43/69] Google Java Format --- .../vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java index e9b37a9e76..35a06b5570 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java @@ -11,7 +11,6 @@ import java.util.Map; /** Responsible for creating k8s formatted docker secret. Parameters are secret and the repo. */ - public class DockerConfigJsonUtils { private static final ObjectMapper objectMapper = new ObjectMapper(); From 8fb13e71f7581f4b25ccfe6bb821f73d405a63fa Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:52:05 +0000 Subject: [PATCH 44/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../src/integration-test/resources/application-test.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties index aba48032ec..aaabe6d21c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties @@ -53,6 +53,9 @@ integrationTest.dataJobsNamespace=${DEPLOYMENT_K8S_NAMESPACE:} integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:} +# If the job builder image is saved in a private docker registry then this +# property should have the name of the secret in the env. +datajobs.builder.registrySecret=${DATAJOBS_BUILDER_REGISTRY_SECRET:} datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL} From 795c0b201cf2eef72d31ae6fce9a9e23268cf627 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 18:58:03 +0000 Subject: [PATCH 45/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../vmware/taurus/authorization/TestKerberosServerHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java index 454f644d90..5e74db5f6d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java @@ -64,6 +64,7 @@ public static void shutdownServer() throws KrbException, IOException { try { Files.delete(KDC_WORK_DIR.toPath()); } catch (Exception e) { + // This error happens occasionally when running tests locally. It is no real concern that the dir is not cleaned up. log.error("Failed to delete test directory.", e); } } From b5d058011d14b1a45f685c0281cd885b18e5082d Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 18:58:30 +0000 Subject: [PATCH 46/69] Google Java Format --- .../vmware/taurus/authorization/TestKerberosServerHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java index 5e74db5f6d..f718e66b41 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java @@ -64,7 +64,8 @@ public static void shutdownServer() throws KrbException, IOException { try { Files.delete(KDC_WORK_DIR.toPath()); } catch (Exception e) { - // This error happens occasionally when running tests locally. It is no real concern that the dir is not cleaned up. + // This error happens occasionally when running tests locally. It is no real concern that the + // dir is not cleaned up. log.error("Failed to delete test directory.", e); } } From 50b5925a5d3008fbcb57801c63108934df81678a Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 21 Nov 2022 20:12:21 +0000 Subject: [PATCH 47/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../it/PrivateBuilderDockerRepoIT.java | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 00e808b768..d236308e7e 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -12,6 +12,7 @@ import com.vmware.taurus.controlplane.model.data.DataJobVersion; import com.vmware.taurus.datajobs.it.common.BaseIT; import com.vmware.taurus.datajobs.it.common.DockerConfigJsonUtils; +import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1SecretBuilder; import lombok.extern.slf4j.Slf4j; @@ -59,24 +60,31 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { private String dataJobsBuilderRegistrySecretContent; public void createBuilderImagePullSecret(String namespaceName) throws Exception { - new CoreV1Api(controlKubernetesService.getClient()) - .createNamespacedSecret( - namespaceName, - new V1SecretBuilder() - .withNewMetadata() - .withName("integration-test-docker-pull-secret") - .withNamespace(namespaceName) - .endMetadata() - .withStringData( - DockerConfigJsonUtils.create( - "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - dataJobsBuilderRegistrySecretContent)) - .withType("kubernetes.io/dockerconfigjson") - .build(), - null, - null, - null, - null); + try { + new CoreV1Api(controlKubernetesService.getClient()) + .createNamespacedSecret( + namespaceName, + new V1SecretBuilder() + .withNewMetadata() + .withName("integration-test-docker-pull-secret") + .withNamespace(namespaceName) + .endMetadata() + .withStringData( + DockerConfigJsonUtils.create( + "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + dataJobsBuilderRegistrySecretContent)) + .withType("kubernetes.io/dockerconfigjson") + .build(), + null, + null, + null, + null); + }catch(ApiException e){ + if(e.getCode() == 409){ + return; + } + throw e; + } } @AfterEach From c5f220c72376365fb5031dd80ad6642767de3324 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 21 Nov 2022 20:12:47 +0000 Subject: [PATCH 48/69] Google Java Format --- .../it/PrivateBuilderDockerRepoIT.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index d236308e7e..ed96fee48d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -62,25 +62,25 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { public void createBuilderImagePullSecret(String namespaceName) throws Exception { try { new CoreV1Api(controlKubernetesService.getClient()) - .createNamespacedSecret( - namespaceName, - new V1SecretBuilder() - .withNewMetadata() - .withName("integration-test-docker-pull-secret") - .withNamespace(namespaceName) - .endMetadata() - .withStringData( - DockerConfigJsonUtils.create( - "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", - dataJobsBuilderRegistrySecretContent)) - .withType("kubernetes.io/dockerconfigjson") - .build(), - null, - null, - null, - null); - }catch(ApiException e){ - if(e.getCode() == 409){ + .createNamespacedSecret( + namespaceName, + new V1SecretBuilder() + .withNewMetadata() + .withName("integration-test-docker-pull-secret") + .withNamespace(namespaceName) + .endMetadata() + .withStringData( + DockerConfigJsonUtils.create( + "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + dataJobsBuilderRegistrySecretContent)) + .withType("kubernetes.io/dockerconfigjson") + .build(), + null, + null, + null, + null); + } catch (ApiException e) { + if (e.getCode() == 409) { return; } throw e; From a38046d417e2807c0a9a33c6768d46d44f4b72b2 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:19:09 +0000 Subject: [PATCH 49/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../main/java/com/vmware/taurus/service/KubernetesService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 3fcc044692..9ae3054c92 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -6,6 +6,7 @@ package com.vmware.taurus.service; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import com.google.common.collect.Iterables; import com.google.gson.JsonSyntaxException; @@ -2505,6 +2506,7 @@ static int convertMemoryToMBs(Quantity quantity) { return quantity.getNumber().toBigInteger().divide(divider.multiply(divider)).intValue(); } + @VisibleForTesting public ApiClient getClient() { return client; } From 32ed5d6f41922b9b94052a0047bc3175397da180 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:20:57 +0000 Subject: [PATCH 50/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 2 +- .../java/com/vmware/taurus/datajobs/it/common/BaseIT.java | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index ed96fee48d..07c66b1f64 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -126,7 +126,7 @@ public void setup() throws Exception { @Test public void testPrivateDockerBuildJob() throws Exception { - createBuilderImagePullSecret(getControlNamespace()); + createBuilderImagePullSecret(controlNamespace); // Take the job zip as byte array byte[] jobZipBinary = IOUtils.toByteArray( diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index d3f808b060..6927ffb775 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -89,13 +89,10 @@ public KerberosCredentialsRepository credentialsRepository() { private boolean ownsDataJobsNamespace = false; @Value("${integrationTest.controlNamespace:}") - private String controlNamespace; + protected String controlNamespace; private boolean ownsControlNamespace = false; - public String getControlNamespace() { - return controlNamespace; - } @BeforeEach public void before() throws Exception { From 169b002af07d865367c6dd1c8880fb1d7d217f40 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 12:21:21 +0000 Subject: [PATCH 51/69] Google Java Format --- .../java/com/vmware/taurus/datajobs/it/common/BaseIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 6927ffb775..5504bb2198 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -93,7 +93,6 @@ public KerberosCredentialsRepository credentialsRepository() { private boolean ownsControlNamespace = false; - @BeforeEach public void before() throws Exception { log.info("Running test with: {} bytes of memory.", Runtime.getRuntime().totalMemory()); From 48f9916bac16ad22886c6c823475faf076bcf1db Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:27:27 +0000 Subject: [PATCH 52/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 07c66b1f64..8d7e712c1c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -145,7 +145,7 @@ public void testPrivateDockerBuildJob() throws Exception { .andReturn(); DataJobVersion testDataJobVersion = - new ObjectMapper() + BaseIT.mapper .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); Assertions.assertNotNull(testDataJobVersion); @@ -188,7 +188,7 @@ public void testPrivateDockerBuildJob() throws Exception { .andReturn(); // wait for pod to initialize - Thread.sleep(4000); + Thread.sleep(4000); // We just don't check when we know the pod is defo not up. It keeps the logs a lot cleaner. Awaitility.await() .atMost(10, TimeUnit.SECONDS) .until( From a33b78082dbc83acc524c1523886a30d21ec4e52 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 12:27:55 +0000 Subject: [PATCH 53/69] Google Java Format --- .../taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 8d7e712c1c..e0bccbe1e5 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -5,7 +5,6 @@ package com.vmware.taurus.datajobs.it; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.internal.LinkedTreeMap; import com.vmware.taurus.ControlplaneApplication; @@ -145,8 +144,8 @@ public void testPrivateDockerBuildJob() throws Exception { .andReturn(); DataJobVersion testDataJobVersion = - BaseIT.mapper - .readValue(jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); + BaseIT.mapper.readValue( + jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class); Assertions.assertNotNull(testDataJobVersion); String testJobVersionSha = testDataJobVersion.getVersionSha(); @@ -188,7 +187,9 @@ public void testPrivateDockerBuildJob() throws Exception { .andReturn(); // wait for pod to initialize - Thread.sleep(4000); // We just don't check when we know the pod is defo not up. It keeps the logs a lot cleaner. + Thread.sleep( + 4000); // We just don't check when we know the pod is defo not up. It keeps the logs a lot + // cleaner. Awaitility.await() .atMost(10, TimeUnit.SECONDS) .until( From bf104715931154073e7733e22666b3afb5a6c4ae Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:37:47 +0000 Subject: [PATCH 54/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../datajobs/it/PrivateBuilderDockerRepoIT.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index e0bccbe1e5..37e0a93803 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -58,7 +58,7 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { @Value("${datajobs.builder.registrySecret.content.testOnly:}") private String dataJobsBuilderRegistrySecretContent; - public void createBuilderImagePullSecret(String namespaceName) throws Exception { + private void createBuilderImagePullSecret(String namespaceName) throws Exception { try { new CoreV1Api(controlKubernetesService.getClient()) .createNamespacedSecret( @@ -123,6 +123,11 @@ public void setup() throws Exception { .andExpect(status().isCreated()); } + + /** + * Within this test we assert only that the data job execution is started and has an execution id. + * We don't wait for the job to be completed as successful as that takes too long + */ @Test public void testPrivateDockerBuildJob() throws Exception { createBuilderImagePullSecret(controlNamespace); @@ -187,9 +192,6 @@ public void testPrivateDockerBuildJob() throws Exception { .andReturn(); // wait for pod to initialize - Thread.sleep( - 4000); // We just don't check when we know the pod is defo not up. It keeps the logs a lot - // cleaner. Awaitility.await() .atMost(10, TimeUnit.SECONDS) .until( @@ -210,7 +212,7 @@ public void testPrivateDockerBuildJob() throws Exception { var gson = new Gson(); ArrayList parsed = gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); - return StringUtils.isNotBlank((String) parsed.get(0).get("id")); + return StringUtils.isNotBlank((String) parsed.get(0).get("id")); // simply check there is an exeution id. We don't care the status of the job } catch (Exception e) { return false; } From dd7cfa9b6fcced2640261fd20a711cca5ea38632 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:40:35 +0000 Subject: [PATCH 55/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 37e0a93803..161103e0e0 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -125,6 +125,10 @@ public void setup() throws Exception { /** + * This test exists to make sure that the builder image can be pulled from a private repo. + * We create, build, deploy a data job and manually start execution to test this. + * We don't wait for the job to complete because that is irrelevant. + * * Within this test we assert only that the data job execution is started and has an execution id. * We don't wait for the job to be completed as successful as that takes too long */ From 86a222a3b4624abd48041bafe7b54c9d16a24478 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 12:41:10 +0000 Subject: [PATCH 56/69] Google Java Format --- .../it/PrivateBuilderDockerRepoIT.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 161103e0e0..947add1dca 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -123,14 +123,13 @@ public void setup() throws Exception { .andExpect(status().isCreated()); } - /** - * This test exists to make sure that the builder image can be pulled from a private repo. - * We create, build, deploy a data job and manually start execution to test this. - * We don't wait for the job to complete because that is irrelevant. + * This test exists to make sure that the builder image can be pulled from a private repo. We + * create, build, deploy a data job and manually start execution to test this. We don't wait for + * the job to complete because that is irrelevant. * - * Within this test we assert only that the data job execution is started and has an execution id. - * We don't wait for the job to be completed as successful as that takes too long + *

Within this test we assert only that the data job execution is started and has an execution + * id. We don't wait for the job to be completed as successful as that takes too long */ @Test public void testPrivateDockerBuildJob() throws Exception { @@ -216,7 +215,13 @@ public void testPrivateDockerBuildJob() throws Exception { var gson = new Gson(); ArrayList parsed = gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class); - return StringUtils.isNotBlank((String) parsed.get(0).get("id")); // simply check there is an exeution id. We don't care the status of the job + return StringUtils.isNotBlank( + (String) + parsed + .get(0) + .get( + "id")); // simply check there is an exeution id. We don't care the + // status of the job } catch (Exception e) { return false; } From 2929b58978418bffe577ba01da72ec7abc2cae6e Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:45:40 +0000 Subject: [PATCH 57/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../taurus/service/deploy/DeploymentService.java | 4 ++-- .../service/deploy/DockerRegistryService.java | 2 +- .../taurus/service/deploy/JobImageBuilder.java | 14 ++------------ .../service/deploy/DeploymentServiceTest.java | 12 ++++++------ 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java index 904afa5a99..2aa101bc87 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java @@ -146,10 +146,10 @@ public void updateDeployment( dockerRegistryService.dataJobImage( jobDeployment.getDataJobName(), jobDeployment.getGitCommitSha()); - String registrySecret = dockerRegistryService.getRegistrySecret(); + if (jobImageBuilder.buildImage( - imageName, dataJob, jobDeployment, sendNotification, registrySecret)) { + imageName, dataJob, jobDeployment, sendNotification)) { log.info( "Image {} has been built. Will now schedule job {} for execution", imageName, diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java index d59b325a47..6ed20b5e78 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java @@ -24,7 +24,7 @@ public String dataJobImage(String dataJobName, String gitCommitSha) { return String.format("%s/%s:%s", proxyRepositoryURL, dataJobName, gitCommitSha); } - public String getRegistrySecret() { + public String registrySecret() { return registrySecret; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index c7adc10e5e..7da7f4c419 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -106,12 +106,6 @@ public JobImageBuilder( this.kubernetesResources = kubernetesResources; } - /** see {@link JobImageBuilder#buildImage(String, DataJob, JobDeployment, Boolean, String)}} */ - public boolean buildImage( - String imageName, DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification) - throws ApiException, IOException, InterruptedException { - return buildImage(imageName, dataJob, jobDeployment, sendNotification, null); - } /** * Builds and pushes a docker image for a data job. Runs a job on k8s which is responsible for * building and pushing the data job image. This call will block until the builder job has @@ -121,8 +115,6 @@ public boolean buildImage( * @param dataJob Information about the data job. * @param jobDeployment Information about the data job deployment. * @param sendNotification - * @param registrySecret should be the name of the secret that exists in the kubernetes - * cluster/namespace already. * @return True if build and push was successful. False otherwise. * @throws ApiException * @throws IOException @@ -132,8 +124,7 @@ public boolean buildImage( String imageName, DataJob dataJob, JobDeployment jobDeployment, - Boolean sendNotification, - String registrySecret) + Boolean sendNotification) throws ApiException, IOException, InterruptedException { log.info("Build data job image for job {}. Image name: {}", dataJob.getName(), imageName); @@ -199,8 +190,7 @@ public boolean buildImage( builderSecurityContextRunAsUser, builderSecurityContextRunAsGroup, builderSecurityContextFsGroup, - builderServiceAccountName, - registrySecret); + builderServiceAccountName, dockerRegistryService.registrySecret()); log.debug( "Waiting for builder job {} for data job version {}", diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java index dbd6170d36..e5877844e0 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/DeploymentServiceTest.java @@ -142,14 +142,14 @@ public void updateDeployment_newDeploymentCreated() when(dockerRegistryService.dataJobImage(TEST_JOB_NAME, "test-commit")) .thenReturn(TEST_JOB_IMAGE_NAME); - when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null)) + when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true)) .thenReturn(true); deploymentService.updateDeployment( testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME, OP_ID); verify(dockerRegistryService).dataJobImage(TEST_JOB_NAME, "test-commit"); - verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null); + verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService) .createCronJob( eq(TEST_CRONJOB_NAME), @@ -188,7 +188,7 @@ public void updateDeployment_existingDeploymentUpdated() when(dockerRegistryService.dataJobImage(TEST_JOB_NAME, "test-commit")) .thenReturn(TEST_JOB_IMAGE_NAME); - when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null)) + when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true)) .thenReturn(true); when(kubernetesService.listCronJobs()).thenReturn(Set.of(TEST_CRONJOB_NAME)); @@ -196,7 +196,7 @@ public void updateDeployment_existingDeploymentUpdated() testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME, OP_ID); verify(dockerRegistryService).dataJobImage(TEST_JOB_NAME, "test-commit"); - verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null); + verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService) .updateCronJob( eq(TEST_CRONJOB_NAME), @@ -232,14 +232,14 @@ public void updateDeployment_failedToBuildImage_deploymentSkipped() when(dockerRegistryService.dataJobImage(TEST_JOB_NAME, "test-commit")) .thenReturn(TEST_JOB_IMAGE_NAME); - when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null)) + when(jobImageBuilder.buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true)) .thenReturn(false); deploymentService.updateDeployment( testDataJob, jobDeployment, true, TEST_PRINCIPAL_NAME, OP_ID); verify(dockerRegistryService).dataJobImage(TEST_JOB_NAME, "test-commit"); - verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true, null); + verify(jobImageBuilder).buildImage(TEST_JOB_IMAGE_NAME, testDataJob, jobDeployment, true); verify(kubernetesService, never()) .updateCronJob( anyString(), From 42812c41c299fcecc9395b98f6b606d6cf8e17ec Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 12:46:12 +0000 Subject: [PATCH 58/69] Google Java Format --- .../taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 5 ++--- .../vmware/taurus/service/deploy/DeploymentService.java | 5 +---- .../com/vmware/taurus/service/deploy/JobImageBuilder.java | 8 +++----- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 947add1dca..8dc1fce2f1 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -219,9 +219,8 @@ public void testPrivateDockerBuildJob() throws Exception { (String) parsed .get(0) - .get( - "id")); // simply check there is an exeution id. We don't care the - // status of the job + .get("id")); // simply check there is an exeution id. We don't care the + // status of the job } catch (Exception e) { return false; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java index 2aa101bc87..1ee811310d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java @@ -146,10 +146,7 @@ public void updateDeployment( dockerRegistryService.dataJobImage( jobDeployment.getDataJobName(), jobDeployment.getGitCommitSha()); - - - if (jobImageBuilder.buildImage( - imageName, dataJob, jobDeployment, sendNotification)) { + if (jobImageBuilder.buildImage(imageName, dataJob, jobDeployment, sendNotification)) { log.info( "Image {} has been built. Will now schedule job {} for execution", imageName, diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index 7da7f4c419..8642a9e304 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -121,10 +121,7 @@ public JobImageBuilder( * @throws InterruptedException */ public boolean buildImage( - String imageName, - DataJob dataJob, - JobDeployment jobDeployment, - Boolean sendNotification) + String imageName, DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification) throws ApiException, IOException, InterruptedException { log.info("Build data job image for job {}. Image name: {}", dataJob.getName(), imageName); @@ -190,7 +187,8 @@ public boolean buildImage( builderSecurityContextRunAsUser, builderSecurityContextRunAsGroup, builderSecurityContextFsGroup, - builderServiceAccountName, dockerRegistryService.registrySecret()); + builderServiceAccountName, + dockerRegistryService.registrySecret()); log.debug( "Waiting for builder job {} for data job version {}", From b7aec33fa27f618b1e5bf857ac1e3f70c781dc27 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 12:46:53 +0000 Subject: [PATCH 59/69] vdk-pipelines-control-service: integration test Signed-off-by: murphp15 --- .../java/com/vmware/taurus/service/KubernetesService.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index 9ae3054c92..7dcd1a5fab 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -1235,13 +1235,12 @@ public void createJob( .runAsGroup(runAsGroup) .fsGroup(fsGroup)); - if (StringUtils.isNotEmpty(registrySecret)) { - podSpecBuilder.addNewImagePullSecret().withName(registrySecret).endImagePullSecret(); - } if (StringUtils.isNotEmpty(serviceAccountName)) { podSpecBuilder.withServiceAccountName(serviceAccountName); } - + if (StringUtils.isNotEmpty(registrySecret)) { + podSpecBuilder.addNewImagePullSecret().withName(registrySecret).endImagePullSecret(); + } var template = new V1PodTemplateSpecBuilder().withSpec(podSpecBuilder.build()).build(); var spec = new V1JobSpecBuilder() From 5c8e55547310e4ae0bd917bc6bb8f5535df32fa1 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 14:29:41 +0000 Subject: [PATCH 60/69] vdk-pipelines-control-service: integration test docs Signed-off-by: murphp15 --- .../src/integration-test/README.md | 11 +++++++++++ .../resources/application-private-builder.properties | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md b/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md index bf1a2b16cf..5bbe66608c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md @@ -23,6 +23,17 @@ datajobs.git.username=${GIT_USERNAME} # self explanatory datajobs.git.password=${GIT_PASSWORD} # a git personal access token datajobs.docker.registrySecret=${DOCKER_REGISTRY_SECRET:} # the secret in k8s regcred ``` +* Fill required values section in [integration-test/resources/application-private-builder.properties](./resources/application-private-builder.properties) +```properties +datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET} # this should be username:password b64 encoded +``` +* the private package manager must contain the job builder image +```bash +docker login --username ${GIT_USERNAME} --password ${GIT_PASSWORD} ${DOCKER_REGISTRY_URL} +docker pull registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 +docker tag registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 +docker push ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 +``` # Run diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties index 90a0abcea9..1478bd3c1e 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -1,3 +1,3 @@ datajobs.builder.registrySecret=integration-test-docker-pull-secret datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET} -datajobs.builder.image=vmwaresaas.jfrog.io/taurus-dev/versatiledatakit/job-builder:1.2.3 +datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 From e725b44ab45d11e7796f1d74075d537403b5608d Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 14:37:04 +0000 Subject: [PATCH 61/69] vdk-pipelines-control-service: integration test docs Signed-off-by: murphp15 --- .../pipelines_control_service/src/integration-test/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md b/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md index 5bbe66608c..93283667ed 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md @@ -35,7 +35,6 @@ docker tag registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 ${DOCKER_R docker push ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 ``` - # Run ## IntelliJ For the ```org.unbroken-dome.test-sets``` plugin to work well with IntelliJ you need Intellij Version ```2019.3 +``` From 51f8851dbe299b69ace85f7f24dd431a413d3bfa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Nov 2022 14:39:30 +0000 Subject: [PATCH 62/69] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../pipelines_control_service/src/integration-test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md b/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md index 93283667ed..9f59c08ff7 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/README.md @@ -32,7 +32,7 @@ datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET} docker login --username ${GIT_USERNAME} --password ${GIT_PASSWORD} ${DOCKER_REGISTRY_URL} docker pull registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 docker tag registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 -docker push ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 +docker push ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 ``` # Run From f865403a55fc2dbcf5682483df16faef2c7d2a8e Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 15:09:12 +0000 Subject: [PATCH 63/69] vdk-pipelines-control-service: integration test docs Signed-off-by: murphp15 --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 8dc1fce2f1..fa3f84d963 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -79,7 +79,7 @@ private void createBuilderImagePullSecret(String namespaceName) throws Exception null, null); } catch (ApiException e) { - if (e.getCode() == 409) { + if (e.getCode() == 409) { // Value already exists in k8s. return; } throw e; From 248b08292d111613af7286685400ccd7281bd328 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 16:01:28 +0000 Subject: [PATCH 64/69] vdk-pipelines-control-service: integration test docs Signed-off-by: murphp15 --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index fa3f84d963..66008dfbb5 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -135,9 +135,7 @@ public void setup() throws Exception { public void testPrivateDockerBuildJob() throws Exception { createBuilderImagePullSecret(controlNamespace); // Take the job zip as byte array - byte[] jobZipBinary = - IOUtils.toByteArray( - getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); + byte[] jobZipBinary = IOUtils.toByteArray(getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); // Execute job upload with user MvcResult jobUploadResult = From c7bf8cee42b2ac1541a91603df5b6a95ba5f4835 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 16:01:56 +0000 Subject: [PATCH 65/69] Google Java Format --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 66008dfbb5..fa3f84d963 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -135,7 +135,9 @@ public void setup() throws Exception { public void testPrivateDockerBuildJob() throws Exception { createBuilderImagePullSecret(controlNamespace); // Take the job zip as byte array - byte[] jobZipBinary = IOUtils.toByteArray(getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); + byte[] jobZipBinary = + IOUtils.toByteArray( + getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip")); // Execute job upload with user MvcResult jobUploadResult = From ab260ef1e113532f04941342ea0e2216df8934ab Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 18:22:14 +0000 Subject: [PATCH 66/69] vdk-pipelines-control-service: integration test docs Signed-off-by: murphp15 --- .../taurus/datajobs/it/common/BaseIT.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 5504bb2198..2912eda18f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -63,16 +63,6 @@ public class BaseIT extends KerberosSecurityTestcaseJunit5 { protected static final ObjectMapper mapper = new ObjectMapper(); - @TestConfiguration - static class KerberosConfig { - - @Bean - @Primary - public KerberosCredentialsRepository credentialsRepository() { - return new MiniKdcCredentialsRepository(); - } - } - @Autowired private MiniKdcCredentialsRepository kerberosCredentialsRepository; @Autowired protected DataJobsKubernetesService dataJobsKubernetesService; @@ -93,6 +83,17 @@ public KerberosCredentialsRepository credentialsRepository() { private boolean ownsControlNamespace = false; + + @TestConfiguration + static class KerberosConfig { + + @Bean + @Primary + public KerberosCredentialsRepository credentialsRepository() { + return new MiniKdcCredentialsRepository(); + } + } + @BeforeEach public void before() throws Exception { log.info("Running test with: {} bytes of memory.", Runtime.getRuntime().totalMemory()); From eca31debe31628f6ef434e3231008e42d170df29 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 18:22:48 +0000 Subject: [PATCH 67/69] Google Java Format --- .../java/com/vmware/taurus/datajobs/it/common/BaseIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java index 2912eda18f..a92f01b1ac 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java @@ -83,7 +83,6 @@ public class BaseIT extends KerberosSecurityTestcaseJunit5 { private boolean ownsControlNamespace = false; - @TestConfiguration static class KerberosConfig { From ecd16b91730ccc2974ff64fa3e4087478c5ea626 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 22 Nov 2022 18:31:27 +0000 Subject: [PATCH 68/69] vdk-pipelines-control-service: integration test docs Signed-off-by: murphp15 --- .../taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index fa3f84d963..8d7786b9d4 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -58,6 +58,9 @@ public class PrivateBuilderDockerRepoIT extends BaseIT { @Value("${datajobs.builder.registrySecret.content.testOnly:}") private String dataJobsBuilderRegistrySecretContent; + @Value("${datajobs.builder.image}") + private String builderImage; + private void createBuilderImagePullSecret(String namespaceName) throws Exception { try { new CoreV1Api(controlKubernetesService.getClient()) @@ -70,7 +73,7 @@ private void createBuilderImagePullSecret(String namespaceName) throws Exception .endMetadata() .withStringData( DockerConfigJsonUtils.create( - "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit", + builderImage.substring(0, builderImage.lastIndexOf("/")), dataJobsBuilderRegistrySecretContent)) .withType("kubernetes.io/dockerconfigjson") .build(), From ff46b38517f7544d3b4f1a955c79d2987d515882 Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 22 Nov 2022 18:32:02 +0000 Subject: [PATCH 69/69] Google Java Format --- .../vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java index 8d7786b9d4..7cfb7f63f7 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java @@ -73,7 +73,7 @@ private void createBuilderImagePullSecret(String namespaceName) throws Exception .endMetadata() .withStringData( DockerConfigJsonUtils.create( - builderImage.substring(0, builderImage.lastIndexOf("/")), + builderImage.substring(0, builderImage.lastIndexOf("/")), dataJobsBuilderRegistrySecretContent)) .withType("kubernetes.io/dockerconfigjson") .build(),