From 137896440c860ca95bc4a64641c5a3bd9617cbee Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 28 Nov 2022 17:09:22 +0000 Subject: [PATCH 1/9] control-service: builder images can load secrets from k8s Why What How was this tested tested extensivly locally and it works very well and leads to faster dev iteration cycles. Signed-off-by: murphp15 --- .../main/java/com/vmware/taurus/service/KubernetesService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index e413be8adf..b875be4fbf 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -2202,6 +2202,7 @@ public static V1Container container( .withRequests(resources(request)) .withLimits(resources(limit)) .build()) + .withEnvFrom(new V1EnvFromSource().secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) .withEnv( envs.entrySet().stream() .map(KubernetesService::envVar) From a198ea576a244fbe0d391a5992f529f933bc3d4a Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Mon, 28 Nov 2022 17:09:49 +0000 Subject: [PATCH 2/9] Google Java Format --- .../java/com/vmware/taurus/service/KubernetesService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index b875be4fbf..49d87ae31f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -2202,7 +2202,9 @@ public static V1Container container( .withRequests(resources(request)) .withLimits(resources(limit)) .build()) - .withEnvFrom(new V1EnvFromSource().secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) + .withEnvFrom( + new V1EnvFromSource() + .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) .withEnv( envs.entrySet().stream() .map(KubernetesService::envVar) From 9fe32286747fde79edde552db17a241872332967 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 28 Nov 2022 17:10:28 +0000 Subject: [PATCH 3/9] control-service: builder images can load secrets from k8s Why What How was this tested tested extensivly locally and it works very well and leads to faster dev iteration cycles. Signed-off-by: murphp15 --- projects/control-service/projects/job-builder/build_image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/control-service/projects/job-builder/build_image.sh b/projects/control-service/projects/job-builder/build_image.sh index 0af4cbba6c..7122159059 100644 --- a/projects/control-service/projects/job-builder/build_image.sh +++ b/projects/control-service/projects/job-builder/build_image.sh @@ -52,6 +52,7 @@ cat > /kaniko/.docker/config.json <<- EOM "password":"$registry_password", "auth": "$auth" } + $extra_auth } } EOM From c9ab0287171b6a33c7a3f9200f0a2cd5f3be7f43 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 28 Nov 2022 17:20:25 +0000 Subject: [PATCH 4/9] control-service: builder images can load secrets from k8s Why What How was this tested tested extensivly locally and it works very well and leads to faster dev iteration cycles. Signed-off-by: murphp15 --- .../resources/application-private-builder.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties index 1478bd3c1e..3371bf343f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -1,3 +1,4 @@ datajobs.builder.registrySecret=integration-test-docker-pull-secret datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET} datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 +datajobs.deployment.dataJobBaseImage=ghcr.io/versatile-data-kit-dev/dp/versatiledatakit/data-job-base-python-3.7:latest From 26ac8f1febaf81f538951d5737345c294b4bd68a Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 28 Nov 2022 19:09:42 +0000 Subject: [PATCH 5/9] control-service: builder images can load secrets from k8s Why What How was this tested tested extensivly locally and it works very well and leads to faster dev iteration cycles. Signed-off-by: murphp15 --- projects/control-service/projects/job-builder/build_image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/control-service/projects/job-builder/build_image.sh b/projects/control-service/projects/job-builder/build_image.sh index 7122159059..ebc2c84043 100644 --- a/projects/control-service/projects/job-builder/build_image.sh +++ b/projects/control-service/projects/job-builder/build_image.sh @@ -15,6 +15,10 @@ registry_type=$8 registry_username=$9 registry_password=${10} +# Within this property docker config should be included to connect to the registry used to pull the image from. +# it should be prefixed with a comma +# example: ,"ghcr.io/versatile-data-kit-dev/dp/versatiledatakit":{"auth":"dmVyc2F0aWxlLWRhdGEta2l0LWRldjo8bXlUb2tlbj4="}} +extra_auth=${extra_auth:-""} # Echo selected data to be logged echo "AWS_REGION=$aws_region" echo "DOCKER_REGISTRY=$docker_registry" From d11cd5087fd16ef41d0ea7453e0520fc3307900f Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 28 Nov 2022 19:11:42 +0000 Subject: [PATCH 6/9] control-service: builder images can load secrets from k8s Why What How was this tested tested extensivly locally and it works very well and leads to faster dev iteration cycles. Signed-off-by: murphp15 --- projects/control-service/projects/base/build.gradle | 1 + .../projects/versions-of-external-dependencies.gradle | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/base/build.gradle b/projects/control-service/projects/base/build.gradle index d057130c39..4cd1a97fa8 100644 --- a/projects/control-service/projects/base/build.gradle +++ b/projects/control-service/projects/base/build.gradle @@ -78,6 +78,7 @@ dependencies { testImplementation versions.'org.awaitility:awaitility' testImplementation versions.'com.github.tomakehurst:wiremock' testImplementation versions.'org.mockito:mockito-core' + testImplementation versions.'net.bytebuddy:byte-buddy' testImplementation 'org.junit.jupiter:junit-jupiter-api' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } diff --git a/projects/control-service/projects/versions-of-external-dependencies.gradle b/projects/control-service/projects/versions-of-external-dependencies.gradle index 25320f5215..7cd9140bf1 100644 --- a/projects/control-service/projects/versions-of-external-dependencies.gradle +++ b/projects/control-service/projects/versions-of-external-dependencies.gradle @@ -18,7 +18,8 @@ project.ext { 'org.junit.jupiter:junit-jupiter-engine' : 'org.junit.jupiter:junit-jupiter-engine:5.7.2', 'org.junit.platform:junit-platform-suite-api' : 'org.junit.platform:junit-platform-suite-api:1.8.1', 'com.mmnaseri.utils:spring-data-mock' : 'com.mmnaseri.utils:spring-data-mock:2.2.0', - 'org.mockito:mockito-core' : 'org.mockito:mockito-core:3.11.2', + 'org.mockito:mockito-core' : 'org.mockito:mockito-core:4.9.0', + 'net.bytebuddy:byte-buddy' : 'net.bytebuddy:byte-buddy:1.12.19', 'com.fasterxml.jackson.core:jackson-databind' : 'com.fasterxml.jackson.core:jackson-databind:2.12.4', 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' : 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.4', 'org.json:json' : 'org.json:json:20210307', From ad79f4f08bc78f15f3330facab7333d30659f808 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Mon, 28 Nov 2022 21:03:45 +0000 Subject: [PATCH 7/9] control-service: builder base image in helm Signed-off-by: murphp15 --- .../projects/pipelines_control_service/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/control-service/projects/pipelines_control_service/build.gradle b/projects/control-service/projects/pipelines_control_service/build.gradle index ca95c09839..2919a99719 100644 --- a/projects/control-service/projects/pipelines_control_service/build.gradle +++ b/projects/control-service/projects/pipelines_control_service/build.gradle @@ -82,6 +82,7 @@ dependencies { // Implementation dependencies are found on compile classpath of testImplementation versions.'org.mock-server:mockserver-netty' testImplementation 'org.springframework.security:spring-security-oauth2-jose' testImplementation versions.'org.mockito:mockito-core' + testImplementation versions.'net.bytebuddy:byte-buddy' testImplementation versions.'org.testcontainers:testcontainers' testImplementation versions.'org.springframework.security.kerberos:spring-security-kerberos-test' testImplementation versions.'org.awaitility:awaitility' From d3a1280a15b57304a78bc728a85998c00d3ca674 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Tue, 29 Nov 2022 09:16:04 +0000 Subject: [PATCH 8/9] control-service: builder base image in helm Signed-off-by: murphp15 --- .../java/com/vmware/taurus/service/KubernetesServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java index 072a4f0860..16256dda0d 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java @@ -54,6 +54,8 @@ public void testCreateVDKContainer() { .build()) .withCommand(vdkCommand) .withArgs(List.of()) + .withEnvFrom(new V1EnvFromSource() + .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) .withEnv(List.of()) .withResources( new V1ResourceRequirementsBuilder() From eff99d0b293eed14d861857cdbef6740f44fbb5f Mon Sep 17 00:00:00 2001 From: github-actions <> Date: Tue, 29 Nov 2022 09:16:38 +0000 Subject: [PATCH 9/9] Google Java Format --- .../com/vmware/taurus/service/KubernetesServiceTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java index 16256dda0d..80c98ca908 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java @@ -54,8 +54,9 @@ public void testCreateVDKContainer() { .build()) .withCommand(vdkCommand) .withArgs(List.of()) - .withEnvFrom(new V1EnvFromSource() - .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) + .withEnvFrom( + new V1EnvFromSource() + .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) .withEnv(List.of()) .withResources( new V1ResourceRequirementsBuilder()