diff --git a/projects/control-service/projects/base/build.gradle b/projects/control-service/projects/base/build.gradle index d057130c39..4cd1a97fa8 100644 --- a/projects/control-service/projects/base/build.gradle +++ b/projects/control-service/projects/base/build.gradle @@ -78,6 +78,7 @@ dependencies { testImplementation versions.'org.awaitility:awaitility' testImplementation versions.'com.github.tomakehurst:wiremock' testImplementation versions.'org.mockito:mockito-core' + testImplementation versions.'net.bytebuddy:byte-buddy' testImplementation 'org.junit.jupiter:junit-jupiter-api' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } diff --git a/projects/control-service/projects/job-builder/build_image.sh b/projects/control-service/projects/job-builder/build_image.sh index 0af4cbba6c..ebc2c84043 100644 --- a/projects/control-service/projects/job-builder/build_image.sh +++ b/projects/control-service/projects/job-builder/build_image.sh @@ -15,6 +15,10 @@ registry_type=$8 registry_username=$9 registry_password=${10} +# Within this property docker config should be included to connect to the registry used to pull the image from. +# it should be prefixed with a comma +# example: ,"ghcr.io/versatile-data-kit-dev/dp/versatiledatakit":{"auth":"dmVyc2F0aWxlLWRhdGEta2l0LWRldjo8bXlUb2tlbj4="}} +extra_auth=${extra_auth:-""} # Echo selected data to be logged echo "AWS_REGION=$aws_region" echo "DOCKER_REGISTRY=$docker_registry" @@ -52,6 +56,7 @@ cat > /kaniko/.docker/config.json <<- EOM "password":"$registry_password", "auth": "$auth" } + $extra_auth } } EOM diff --git a/projects/control-service/projects/pipelines_control_service/build.gradle b/projects/control-service/projects/pipelines_control_service/build.gradle index ca95c09839..2919a99719 100644 --- a/projects/control-service/projects/pipelines_control_service/build.gradle +++ b/projects/control-service/projects/pipelines_control_service/build.gradle @@ -82,6 +82,7 @@ dependencies { // Implementation dependencies are found on compile classpath of testImplementation versions.'org.mock-server:mockserver-netty' testImplementation 'org.springframework.security:spring-security-oauth2-jose' testImplementation versions.'org.mockito:mockito-core' + testImplementation versions.'net.bytebuddy:byte-buddy' testImplementation versions.'org.testcontainers:testcontainers' testImplementation versions.'org.springframework.security.kerberos:spring-security-kerberos-test' testImplementation versions.'org.awaitility:awaitility' diff --git a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties index 1478bd3c1e..3371bf343f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties +++ b/projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties @@ -1,3 +1,4 @@ datajobs.builder.registrySecret=integration-test-docker-pull-secret datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET} datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3 +datajobs.deployment.dataJobBaseImage=ghcr.io/versatile-data-kit-dev/dp/versatiledatakit/data-job-base-python-3.7:latest diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java index e413be8adf..49d87ae31f 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java @@ -2202,6 +2202,9 @@ public static V1Container container( .withRequests(resources(request)) .withLimits(resources(limit)) .build()) + .withEnvFrom( + new V1EnvFromSource() + .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) .withEnv( envs.entrySet().stream() .map(KubernetesService::envVar) diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java index 072a4f0860..80c98ca908 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/KubernetesServiceTest.java @@ -54,6 +54,9 @@ public void testCreateVDKContainer() { .build()) .withCommand(vdkCommand) .withArgs(List.of()) + .withEnvFrom( + new V1EnvFromSource() + .secretRef(new V1SecretEnvSource().name("builder-secrets").optional(true))) .withEnv(List.of()) .withResources( new V1ResourceRequirementsBuilder() diff --git a/projects/control-service/projects/versions-of-external-dependencies.gradle b/projects/control-service/projects/versions-of-external-dependencies.gradle index 25320f5215..7cd9140bf1 100644 --- a/projects/control-service/projects/versions-of-external-dependencies.gradle +++ b/projects/control-service/projects/versions-of-external-dependencies.gradle @@ -18,7 +18,8 @@ project.ext { 'org.junit.jupiter:junit-jupiter-engine' : 'org.junit.jupiter:junit-jupiter-engine:5.7.2', 'org.junit.platform:junit-platform-suite-api' : 'org.junit.platform:junit-platform-suite-api:1.8.1', 'com.mmnaseri.utils:spring-data-mock' : 'com.mmnaseri.utils:spring-data-mock:2.2.0', - 'org.mockito:mockito-core' : 'org.mockito:mockito-core:3.11.2', + 'org.mockito:mockito-core' : 'org.mockito:mockito-core:4.9.0', + 'net.bytebuddy:byte-buddy' : 'net.bytebuddy:byte-buddy:1.12.19', 'com.fasterxml.jackson.core:jackson-databind' : 'com.fasterxml.jackson.core:jackson-databind:2.12.4', 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' : 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.4', 'org.json:json' : 'org.json:json:20210307',