Read builder image from private docker repo

projects/control-service/projects/pipelines_control_service/src/integration-test/README.md

@@ -23,7 +23,17 @@ datajobs.git.username=${GIT_USERNAME} # self explanatory
 datajobs.git.password=${GIT_PASSWORD} # a git personal access token
 datajobs.docker.registrySecret=${DOCKER_REGISTRY_SECRET:} # the secret in k8s regcred
 ```
-
+* Fill required values section in [integration-test/resources/application-private-builder.properties](./resources/application-private-builder.properties)
+```properties
+datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET}  # this should be username:password b64 encoded
+```
+* the private package manager must contain the job builder image
+```bash
+docker login --username ${GIT_USERNAME} --password ${GIT_PASSWORD} ${DOCKER_REGISTRY_URL}
+docker pull registry.hub.docker.com/versatiledatakit/job-builder:1.2.3
+docker tag registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
+docker push ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
+```
 
 # Run
 ## IntelliJ

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/PrivateBuilderDockerRepoIT.java

@@ -0,0 +1,229 @@
+/*
+ * Copyright 2021 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.datajobs.it;
+
+import com.google.gson.Gson;
+import com.google.gson.internal.LinkedTreeMap;
+import com.vmware.taurus.ControlplaneApplication;
+import com.vmware.taurus.controlplane.model.data.DataJobVersion;
+import com.vmware.taurus.datajobs.it.common.BaseIT;
+import com.vmware.taurus.datajobs.it.common.DockerConfigJsonUtils;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.models.V1SecretBuilder;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.platform.commons.util.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Import;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.web.servlet.MvcResult;
+
+import java.util.ArrayList;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@Slf4j
+@Import({DataJobDeploymentCrudIT.TaskExecutorConfig.class})
+@TestPropertySource(
+    properties = {
+      "dataJob.readOnlyRootFileSystem=true",
+    })
+@ActiveProfiles({"test", "private-builder"})
+@SpringBootTest(
+    webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
+    classes = ControlplaneApplication.class)
+public class PrivateBuilderDockerRepoIT extends BaseIT {
+
+  private static final String TEST_JOB_NAME =
+      "private-docker-builder-test-" + UUID.randomUUID().toString().substring(0, 8);
+  private static final Object DEPLOYMENT_ID = "private-docker-builder";
+
+  @Value("${datajobs.builder.registrySecret.content.testOnly:}")
+  private String dataJobsBuilderRegistrySecretContent;
+
+  private void createBuilderImagePullSecret(String namespaceName) throws Exception {
+    try {
+      new CoreV1Api(controlKubernetesService.getClient())
+          .createNamespacedSecret(
+              namespaceName,
+              new V1SecretBuilder()
+                  .withNewMetadata()
+                  .withName("integration-test-docker-pull-secret")
+                  .withNamespace(namespaceName)
+                  .endMetadata()
+                  .withStringData(
+                      DockerConfigJsonUtils.create(
+                          "vmwaresaas.jfrog.io/taurus-dev/versatiledatakit",
+                          dataJobsBuilderRegistrySecretContent))
+                  .withType("kubernetes.io/dockerconfigjson")
+                  .build(),
+              null,
+              null,
+              null,
+              null);
+    } catch (ApiException e) {
+      if (e.getCode() == 409) { // Value already exists in k8s.
+        return;
+      }
+      throw e;
+    }
+  }
+
+  @AfterEach
+  public void cleanUp() throws Exception {
+    // delete job
+    mockMvc
+        .perform(
+            delete(
+                    String.format(
+                        "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME))
+                .with(user("user")))
+        .andExpect(status().isOk());
+
+    // Execute delete deployment
+    mockMvc
+        .perform(
+            delete(
+                    String.format(
+                        "/data-jobs/for-team/%s/jobs/%s/deployments/%s",
+                        TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID))
+                .with(user("user"))
+                .contentType(MediaType.APPLICATION_JSON))
+        .andExpect(status().isAccepted());
+  }
+
+  @BeforeEach
+  public void setup() throws Exception {
+    String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME);
+
+    // Execute create job
+    mockMvc
+        .perform(
+            post(String.format("/data-jobs/for-team/%s/jobs", TEST_TEAM_NAME))
+                .with(user("user"))
+                .content(dataJobRequestBody)
+                .contentType(MediaType.APPLICATION_JSON))
+        .andExpect(status().isCreated());
+  }
+
+  /**
+   * This test exists to make sure that the builder image can be pulled from a private repo. We
+   * create, build, deploy a data job and manually start execution to test this. We don't wait for
+   * the job to complete because that is irrelevant.
+   *
+   * <p>Within this test we assert only that the data job execution is started and has an execution
+   * id. We don't wait for the job to be completed as successful as that takes too long
+   */
+  @Test
+  public void testPrivateDockerBuildJob() throws Exception {
+    createBuilderImagePullSecret(controlNamespace);
+    // Take the job zip as byte array
+    byte[] jobZipBinary =
+        IOUtils.toByteArray(
+            getClass().getClassLoader().getResourceAsStream("job_ephemeral_storage.zip"));
+
+    // Execute job upload with user
+    MvcResult jobUploadResult =
+        mockMvc
+            .perform(
+                post(String.format(
+                        "/data-jobs/for-team/%s/jobs/%s/sources", TEST_TEAM_NAME, TEST_JOB_NAME))
+                    .with(user("user"))
+                    .content(jobZipBinary)
+                    .contentType(MediaType.APPLICATION_OCTET_STREAM))
+            .andExpect(status().isOk())
+            .andReturn();
+
+    DataJobVersion testDataJobVersion =
+        BaseIT.mapper.readValue(
+            jobUploadResult.getResponse().getContentAsString(), DataJobVersion.class);
+    Assertions.assertNotNull(testDataJobVersion);
+
+    String testJobVersionSha = testDataJobVersion.getVersionSha();
+    Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha));
+
+    // Setup
+    String dataJobDeploymentRequestBody = getDataJobDeploymentRequestBody(testJobVersionSha);
+
+    // Execute build and deploy job
+    mockMvc
+        .perform(
+            post(String.format(
+                    "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME))
+                .with(user("user"))
+                .content(dataJobDeploymentRequestBody)
+                .contentType(MediaType.APPLICATION_JSON))
+        .andExpect(status().isAccepted())
+        .andReturn();
+
+    String opId = TEST_JOB_NAME + UUID.randomUUID().toString().toLowerCase();
+
+    // manually start job execution
+    mockMvc
+        .perform(
+            post(String.format(
+                    "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions",
+                    TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID))
+                .with(user("user"))
+                .header(HEADER_X_OP_ID, opId)
+                .contentType(MediaType.APPLICATION_JSON)
+                .content(
+                    "{\n"
+                        + "  \"args\": {\n"
+                        + "    \"key\": \"value\"\n"
+                        + "  },\n"
+                        + "  \"started_by\": \"schedule/runtime\"\n"
+                        + "}"))
+        .andExpect(status().is(202))
+        .andReturn();
+
+    // wait for pod to initialize
+    Awaitility.await()
+        .atMost(10, TimeUnit.SECONDS)
+        .until(
+            () -> {
+              try {
+                // retrieve running job execution id.
+                var exc =
+                    mockMvc
+                        .perform(
+                            get(String.format(
+                                    "/data-jobs/for-team/%s/jobs/%s/deployments/%s/executions",
+                                    TEST_TEAM_NAME, TEST_JOB_NAME, TEST_JOB_DEPLOYMENT_ID))
+                                .with(user("user"))
+                                .contentType(MediaType.APPLICATION_JSON))
+                        .andExpect(status().isOk())
+                        .andReturn();
+
+                var gson = new Gson();
+                ArrayList<LinkedTreeMap> parsed =
+                    gson.fromJson(exc.getResponse().getContentAsString(), ArrayList.class);
+                return StringUtils.isNotBlank(
+                    (String)
+                        parsed
+                            .get(0)
+                            .get("id")); // simply check there is an exeution id. We don't care the
+                // status of the job
+              } catch (Exception e) {
+                return false;
+              }
+            });
+  }
+}

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/BaseIT.java

@@ -52,7 +52,6 @@
 @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
 @ExtendWith(WebHookServerMockExtension.class)
 public class BaseIT extends KerberosSecurityTestcaseJunit5 {
-
   private static Logger log = LoggerFactory.getLogger(BaseIT.class);
 
   public static final String TEST_JOB_SCHEDULE = "15 10 * * *";
@@ -90,7 +89,7 @@ public KerberosCredentialsRepository credentialsRepository() {
   private boolean ownsDataJobsNamespace = false;
 
   @Value("${integrationTest.controlNamespace:}")
-  private String controlNamespace;
+  protected String controlNamespace;
 
   private boolean ownsControlNamespace = false;
 

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/common/DockerConfigJsonUtils.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2021 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.datajobs.it.common;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.Map;
+
+/** Responsible for creating k8s formatted docker secret. Parameters are secret and the repo. */
+public class DockerConfigJsonUtils {
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  public static Map<String, String> create(String repo, String secret)
+      throws JsonProcessingException {
+    return Map.of(
+        ".dockerconfigjson",
+        objectMapper.writeValueAsString(Map.of("auths", Map.of(repo, Map.of("auth", secret)))));
+  }
+}

projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties

@@ -0,0 +1,3 @@
+datajobs.builder.registrySecret=integration-test-docker-pull-secret
+datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET}
+datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3

projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties

@@ -52,6 +52,11 @@ spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://test
 integrationTest.dataJobsNamespace=${DEPLOYMENT_K8S_NAMESPACE:}
 integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:}
 
+
+# If the job builder image is saved in a private docker registry then this
+# property should have the name of the secret in the env.
+datajobs.builder.registrySecret=${DATAJOBS_BUILDER_REGISTRY_SECRET:}
+
 datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3
 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL}
 datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java

@@ -6,6 +6,7 @@
 package com.vmware.taurus.service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Charsets;
 import com.google.common.collect.Iterables;
 import com.google.gson.JsonSyntaxException;
@@ -1206,7 +1207,8 @@ public void createJob(
       long runAsUser,
       long runAsGroup,
       long fsGroup,
-      String serviceAccountName)
+      String serviceAccountName,
+      String registrySecret)
       throws ApiException {
 
     log.debug("Creating k8s job name:{}, image:{}", name, image);
@@ -1236,7 +1238,9 @@ public void createJob(
     if (StringUtils.isNotEmpty(serviceAccountName)) {
       podSpecBuilder.withServiceAccountName(serviceAccountName);
     }
-
+    if (StringUtils.isNotEmpty(registrySecret)) {
+      podSpecBuilder.addNewImagePullSecret().withName(registrySecret).endImagePullSecret();
+    }
     var template = new V1PodTemplateSpecBuilder().withSpec(podSpecBuilder.build()).build();
     var spec =
         new V1JobSpecBuilder()
@@ -2500,4 +2504,9 @@ static int convertMemoryToMBs(Quantity quantity) {
     }
     return quantity.getNumber().toBigInteger().divide(divider.multiply(divider)).intValue();
   }
+
+  @VisibleForTesting
+  public ApiClient getClient() {
+    return client;
+  }
 }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java

@@ -17,10 +17,17 @@ public class DockerRegistryService {
   @Value("${datajobs.builder.image}")
   private String builderImage;
 
+  @Value("${datajobs.builder.registrySecret:}")
+  private String registrySecret;
+
   public String dataJobImage(String dataJobName, String gitCommitSha) {
     return String.format("%s/%s:%s", proxyRepositoryURL, dataJobName, gitCommitSha);
   }
 
+  public String registrySecret() {
+    return registrySecret;
+  }
+
   public String builderImage() {
     return builderImage;
   }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java

@@ -187,7 +187,8 @@ public boolean buildImage(
         builderSecurityContextRunAsUser,
         builderSecurityContextRunAsGroup,
         builderSecurityContextFsGroup,
-        builderServiceAccountName);
+        builderServiceAccountName,
+        dockerRegistryService.registrySecret());
 
     log.debug(
         "Waiting for builder job {} for data job version {}",

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/authorization/TestKerberosServerHelper.java

@@ -61,6 +61,12 @@ public class TestKerberosServerHelper {
   public static void shutdownServer() throws KrbException, IOException {
     simpleKdcServer.stop();
     Files.delete(KEYTAB.toPath());
-    Files.delete(KDC_WORK_DIR.toPath());
+    try {
+      Files.delete(KDC_WORK_DIR.toPath());
+    } catch (Exception e) {
+      // This error happens occasionally when running tests locally. It is no real concern that the
+      // dir is not cleaned up.
+      log.error("Failed to delete test directory.", e);
+    }
   }
 }

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/datajobs/MockKubernetes.java

@@ -167,6 +167,7 @@ private void mockKubernetesService(KubernetesService mock)
             anyLong(),
             anyLong(),
             anyLong(),
+            anyString(),
             anyString());
     doAnswer(inv -> jobs.keySet()).when(mock).listCronJobs();
     doAnswer(inv -> jobs.remove(inv.getArgument(0))).when(mock).deleteJob(anyString());