control-service: builder images load secrets from k8s (#1358)

Why I wrote this PR with the intent of being able to pass extra docker creds to to the image builder so that we would be able to pull a private base image from a docker repo. The builder now contains login details for 2 registries. The registry it pushes to and the registry it pulls base image from. However this PR also introduces the concept of builder secrets being sourced from k8s which leads to much cleaner separation between the control plane and the builder image. I think this will lead to much better support for custom build images in the future. I'm not sure where to add the documentation about the builder-secrets. So I created a ticket to follow up on this later. #1357 How was this been tested tested extensively locally. I added a test to check that the base image can be pulled from a private repo but in fact it is pulling from the same repo as it is pushing to and so it is could let bugs through related to the auth not working for the base image repo. Signed-off-by: murphp15 <[email protected]> * Google Java Format Signed-off-by: murphp15 <[email protected]> Co-authored-by: github-actions <>
vmware · Nov 29, 2022 · 152f772 · 152f772
1 parent 50fbbb3
commit 152f772
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 1 deletion.
diff --git a/projects/control-service/projects/base/build.gradle b/projects/control-service/projects/base/build.gradle
@@ -78,6 +78,7 @@ dependencies {
    testImplementation versions.'org.awaitility:awaitility'
    testImplementation versions.'com.github.tomakehurst:wiremock'
    testImplementation versions.'org.mockito:mockito-core'
+   testImplementation versions.'net.bytebuddy:byte-buddy'
    testImplementation 'org.junit.jupiter:junit-jupiter-api'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
 }
diff --git a/projects/control-service/projects/job-builder/build_image.sh b/projects/control-service/projects/job-builder/build_image.sh
@@ -15,6 +15,10 @@ registry_type=$8
 registry_username=$9
 registry_password=${10}
 
+# Within this property docker config should be included to connect to the registry used to pull the image from.
+# it should be prefixed with a comma
+# example: ,"ghcr.io/versatile-data-kit-dev/dp/versatiledatakit":{"auth":"dmVyc2F0aWxlLWRhdGEta2l0LWRldjo8bXlUb2tlbj4="}}
+extra_auth=${extra_auth:-""}
 # Echo selected data to be logged
 echo "AWS_REGION=$aws_region"
 echo "DOCKER_REGISTRY=$docker_registry"
@@ -52,6 +56,7 @@ cat > /kaniko/.docker/config.json <<- EOM
       "password":"$registry_password",
       "auth": "$auth"
     }
+  $extra_auth
   }
 }
 EOM

diff --git a/projects/control-service/projects/pipelines_control_service/build.gradle b/projects/control-service/projects/pipelines_control_service/build.gradle
@@ -82,6 +82,7 @@ dependencies { // Implementation dependencies are found on compile classpath of
     testImplementation versions.'org.mock-server:mockserver-netty'
     testImplementation 'org.springframework.security:spring-security-oauth2-jose'
     testImplementation versions.'org.mockito:mockito-core'
+    testImplementation versions.'net.bytebuddy:byte-buddy'
     testImplementation versions.'org.testcontainers:testcontainers'
     testImplementation versions.'org.springframework.security.kerberos:spring-security-kerberos-test'
     testImplementation versions.'org.awaitility:awaitility'

diff --git a/...nes_control_service/src/integration-test/resources/application-private-builder.properties b/...nes_control_service/src/integration-test/resources/application-private-builder.properties
@@ -1,3 +1,4 @@
 datajobs.builder.registrySecret=integration-test-docker-pull-secret
 datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET}
 datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
+datajobs.deployment.dataJobBaseImage=ghcr.io/versatile-data-kit-dev/dp/versatiledatakit/data-job-base-python-3.7:latest
diff --git a/.../pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/.../pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java
@@ -2202,6 +2202,9 @@ public static V1Container container(
                     .withRequests(resources(request))
                     .withLimits(resources(limit))
                     .build())
+            .withEnvFrom(
+                new V1EnvFromSource()
+                    .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true)))
             .withEnv(
                 envs.entrySet().stream()
                     .map(KubernetesService::envVar)

diff --git a/...elines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java b/...elines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java
@@ -54,6 +54,9 @@ public void testCreateVDKContainer() {
                     .build())
             .withCommand(vdkCommand)
             .withArgs(List.of())
+            .withEnvFrom(
+                new V1EnvFromSource()
+                    .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true)))
             .withEnv(List.of())
             .withResources(
                 new V1ResourceRequirementsBuilder()

diff --git a/projects/control-service/projects/versions-of-external-dependencies.gradle b/projects/control-service/projects/versions-of-external-dependencies.gradle
@@ -18,7 +18,8 @@ project.ext {
             'org.junit.jupiter:junit-jupiter-engine'                             : 'org.junit.jupiter:junit-jupiter-engine:5.7.2',
             'org.junit.platform:junit-platform-suite-api'                        : 'org.junit.platform:junit-platform-suite-api:1.8.1',
             'com.mmnaseri.utils:spring-data-mock'                                : 'com.mmnaseri.utils:spring-data-mock:2.2.0',
-            'org.mockito:mockito-core'                                           : 'org.mockito:mockito-core:3.11.2',
+            'org.mockito:mockito-core'                                           : 'org.mockito:mockito-core:4.9.0',
+            'net.bytebuddy:byte-buddy'                                           : 'net.bytebuddy:byte-buddy:1.12.19',
             'com.fasterxml.jackson.core:jackson-databind'                        : 'com.fasterxml.jackson.core:jackson-databind:2.12.4',
             'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'             : 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.4',
             'org.json:json'                                                      : 'org.json:json:20210307',