control-service: builder images load secrets from k8s

projects/control-service/projects/base/build.gradle

@@ -78,6 +78,7 @@ dependencies {
    testImplementation versions.'org.awaitility:awaitility'
    testImplementation versions.'com.github.tomakehurst:wiremock'
    testImplementation versions.'org.mockito:mockito-core'
+   testImplementation versions.'net.bytebuddy:byte-buddy'
    testImplementation 'org.junit.jupiter:junit-jupiter-api'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
 }

projects/control-service/projects/job-builder/build_image.sh

@@ -15,6 +15,10 @@ registry_type=$8
 registry_username=$9
 registry_password=${10}
 
+# Within this property docker config should be included to connect to the registry used to pull the image from.
+# it should be prefixed with a comma
+# example: ,"ghcr.io/versatile-data-kit-dev/dp/versatiledatakit":{"auth":"dmVyc2F0aWxlLWRhdGEta2l0LWRldjo8bXlUb2tlbj4="}}
+extra_auth=${extra_auth:-""}
 # Echo selected data to be logged
 echo "AWS_REGION=$aws_region"
 echo "DOCKER_REGISTRY=$docker_registry"
@@ -52,6 +56,7 @@ cat > /kaniko/.docker/config.json <<- EOM
       "password":"$registry_password",
       "auth": "$auth"
     }
+  $extra_auth
   }
 }
 EOM

projects/control-service/projects/pipelines_control_service/build.gradle

@@ -82,6 +82,7 @@ dependencies { // Implementation dependencies are found on compile classpath of
     testImplementation versions.'org.mock-server:mockserver-netty'
     testImplementation 'org.springframework.security:spring-security-oauth2-jose'
     testImplementation versions.'org.mockito:mockito-core'
+    testImplementation versions.'net.bytebuddy:byte-buddy'
     testImplementation versions.'org.testcontainers:testcontainers'
     testImplementation versions.'org.springframework.security.kerberos:spring-security-kerberos-test'
     testImplementation versions.'org.awaitility:awaitility'

projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties

@@ -1,3 +1,4 @@
 datajobs.builder.registrySecret=integration-test-docker-pull-secret
 datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET}
 datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
+datajobs.deployment.dataJobBaseImage=ghcr.io/versatile-data-kit-dev/dp/versatiledatakit/data-job-base-python-3.7:latest

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java

@@ -2202,6 +2202,9 @@ public static V1Container container(
                     .withRequests(resources(request))
                     .withLimits(resources(limit))
                     .build())
+            .withEnvFrom(
+                new V1EnvFromSource()
+                    .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true)))
             .withEnv(
                 envs.entrySet().stream()
                     .map(KubernetesService::envVar)

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java

@@ -54,6 +54,9 @@ public void testCreateVDKContainer() {
                     .build())
             .withCommand(vdkCommand)
             .withArgs(List.of())
+            .withEnvFrom(
+                new V1EnvFromSource()
+                    .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true)))
             .withEnv(List.of())
             .withResources(
                 new V1ResourceRequirementsBuilder()

projects/control-service/projects/versions-of-external-dependencies.gradle

@@ -18,7 +18,8 @@ project.ext {
             'org.junit.jupiter:junit-jupiter-engine'                             : 'org.junit.jupiter:junit-jupiter-engine:5.7.2',
             'org.junit.platform:junit-platform-suite-api'                        : 'org.junit.platform:junit-platform-suite-api:1.8.1',
             'com.mmnaseri.utils:spring-data-mock'                                : 'com.mmnaseri.utils:spring-data-mock:2.2.0',
-            'org.mockito:mockito-core'                                           : 'org.mockito:mockito-core:3.11.2',
+            'org.mockito:mockito-core'                                           : 'org.mockito:mockito-core:4.9.0',
+            'net.bytebuddy:byte-buddy'                                           : 'net.bytebuddy:byte-buddy:1.12.19',
             'com.fasterxml.jackson.core:jackson-databind'                        : 'com.fasterxml.jackson.core:jackson-databind:2.12.4',
             'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'             : 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.4',
             'org.json:json'                                                      : 'org.json:json:20210307',