job-builder: introduced secure base-job-image

projects/control-service/cicd/.gitlab-ci.yml

@@ -140,6 +140,25 @@ control_service_publish_job_base_image:
     changes:
       - projects/control-service/projects/helm_charts/pipelines-control-service/version.txt
 
+control_service_publish_job_base_image-secure:
+  extends: .images:dind
+  stage: publish_artifacts
+  script:
+    - apk add --no-cache bash
+    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
+    - cd projects/control-service/projects/job-base-image-secure
+    - export VERSION_TAG="1.$CI_PIPELINE_ID"
+    - bash -ex ./publish-job-base.sh
+  retry: !reference [.control_service_retry, retry_options]
+  only:
+    refs:
+      - main
+    changes:
+      - projects/control-service/projects/job-base-image-secure/**/*
+  except:
+    changes:
+      - projects/control-service/projects/helm_charts/pipelines-control-service/version.txt
+
 
 control_service_publish_job_builder_image:
   extends: .images:dind

projects/control-service/projects/job-base-image-secure/Dockerfile-data-job-base

@@ -0,0 +1,14 @@
+# https://docs.docker.com/develop/develop-images/dockerfile_best-practices
+FROM photon:latest
+
+# Set the working directory
+WORKDIR /job
+
+# Install python
+RUN yum update -y
+RUN yum install python3-3.10.0-9.ph4 python3-pip-3.10.0-9.ph4 shadow -y
+RUN ln -fs /usr/bin/python3 /usr/local/bin/python
+
+# Install native dependencies so that requirements in requirements.txt can be installed
+# some (like openssl) should be pre-installed in the base image but let's be explicit
+RUN yum install build-essential -y

projects/control-service/projects/job-base-image-secure/README.md

@@ -0,0 +1,17 @@
+# Job base image
+
+Job base image is the container "base" image used when building per data job custom image during deployment.
+
+This directory provides the source of some base images for standard python versions.
+It's used by secured installation of VDK.
+
+The current base image installs supporting libraries for some native bindings necessary for installing from source
+some python packages which user may specify for their data job.
+
+## Build
+
+To build the job_base images run `./publish-job-base` which will publish new base image to versatiledatakit container registry.
+
+## Use
+
+It's then set in values.yaml of the helm chart as `deploymentDataJobBaseImage` option

projects/control-service/projects/job-base-image-secure/publish-job-base.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+VERSION_TAG="${VERSION_TAG:-"0.1dev"}"
+VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"}
+
+function build_and_push_image() {
+    name="$1"
+    docker_file="$2"
+    arguments="$3"
+
+    image_repo="$VDK_DOCKER_REGISTRY_URL/$name"
+    image_tag="$image_repo:$VERSION_TAG"
+
+    docker build -t "$image_tag" -t "$image_repo:latest" -f "$SCRIPT_DIR/$docker_file" $arguments "$SCRIPT_DIR"
+    docker push "$image_tag"
+    docker push "$image_repo:latest"
+}
+
+build_and_push_image \
+    "data-job-base-python-3.10-secure" \
+    Dockerfile-data-job-base