control-service: split build job base image CI/CD step

.gitlab-ci.yml

@@ -15,6 +15,8 @@ include:
 
 stages:
   - build
+  - publish_python_image
+  - publish_job_base_image
   - publish_artifacts
   - pre_release
   - pre_release_image

projects/control-service/cicd/.gitlab-ci.yml

@@ -145,9 +145,9 @@ control_service_publish_job_base_image:
       changes:
         - projects/control-service/projects/job-base-image/**/*
 
-control_service_publish_job_base_image-secure:
+.control_service_publish_job_base_image_secure:
   extends: .images:dind
-  stage: publish_artifacts
+  stage: publish_job_base_image
   script:
     - apk add --no-cache bash
     - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
@@ -160,14 +160,83 @@ control_service_publish_job_base_image-secure:
         && tar -xvf ds.tar.gz
         && mv dist_linux/docker-slim /usr/local/bin/
         && mv dist_linux/docker-slim-sensor /usr/local/bin/
-    - bash -ex ./publish-job-base.sh
+    - bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR
   retry: !reference [.control_service_retry, retry_options]
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       when: never
     - if: '$CI_COMMIT_BRANCH == "main"'
       changes:
         - projects/control-service/projects/job-base-image-secure/**/*
+        - projects/control-service/projects/python-image-secure/**/*
+
+control_service_publish_job_base_image_secure_3_8:
+  extends: .control_service_publish_job_base_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 8
+
+control_service_publish_job_base_image_secure_3_9:
+  extends: .control_service_publish_job_base_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 9
+
+control_service_publish_job_base_image_secure_3_10:
+  extends: .control_service_publish_job_base_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 10
+
+control_service_publish_job_base_image_secure_3_11:
+  extends: .control_service_publish_job_base_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 11
+
+.control_service_publish_python_image_secure:
+  extends: .images:dind
+  stage: publish_python_image
+  script:
+    - apk add --no-cache bash
+    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
+    - cd projects/control-service/projects/python-image-secure
+    - export VERSION_TAG="1.$CI_PIPELINE_ID"
+    - bash -ex ./publish-python-image.sh $PYTHON_MAJOR $PYTHON_MINOR $PYTHON_PATCH
+  retry: !reference [.control_service_retry, retry_options]
+  only:
+    refs:
+      - external_pull_requests
+    changes:
+      - projects/control-service/projects/python-image-secure/**/*
+
+control_service_publish_python_image_secure_3_8:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 8
+    PYTHON_PATCH: 16
+
+control_service_publish_python_image_secure_3_9:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 9
+    PYTHON_PATCH: 16
+
+control_service_publish_python_image_secure_3_10:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 10
+    PYTHON_PATCH: 11
+
+control_service_publish_python_image_secure_3_11:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 11
+    PYTHON_PATCH: 3
 
 control_service_publish_job_builder_image:
   extends: .images:dind

projects/control-service/projects/job-base-image-secure/Dockerfile-data-job-base

@@ -16,3 +16,15 @@ RUN yum erase toybox -y
 
 # Install native dependencies
 RUN yum install shadow build-essential -y
+
+# Install the native dependencies necessary for oracledb python library
+# See https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html
+RUN set -ex \
+      && echo "Installing native dependencies related to support for oracledb python library ..." \
+      && mkdir -p /opt/lib/native  \
+      && yum -y install libaio curl unzip \
+      && curl --insecure --output oracle-instantclient.zip https://download.oracle.com/otn_software/linux/instantclient/2110000/instantclient-basic-linux.x64-21.10.0.0.0dbru.zip \
+      && unzip oracle-instantclient.zip -d /opt/lib/native/oracle && rm -f oracle-instantclient.zip \
+      && sh -c "echo /opt/lib/native/oracle/instantclient_21_10 > /etc/ld.so.conf.d/oracle-instantclient.conf" \
+      && ldconfig \
+      && yum remove -y curl unzip

projects/control-service/projects/job-base-image-secure/publish-job-base-image.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+VERSION_TAG="${VERSION_TAG:-"0.1dev"}"
+VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"}
+
+PYTHON_MAJOR=$1
+PYTHON_MINOR=$2
+python_name="python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+data_job_base_name="data-job-base-python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+data_job_base_docker_file="Dockerfile-data-job-base"
+
+python_image_repo="$VDK_DOCKER_REGISTRY_URL/$python_name"
+python_image_tag_latest="$python_image_repo:latest"
+
+data_job_base_image_repo="$VDK_DOCKER_REGISTRY_URL/$data_job_base_name"
+data_job_base_image_tag_local="$data_job_base_image_repo:local"
+data_job_base_image_tag_version="$data_job_base_image_repo:$VERSION_TAG"
+data_job_base_image_tag_latest="$data_job_base_image_repo:latest"
+
+docker build -t "$data_job_base_image_tag_local" -f "$SCRIPT_DIR/$data_job_base_docker_file" "$SCRIPT_DIR" \
+--build-arg base_image="$python_image_tag_latest"
+
+docker-slim build \
+--target "$data_job_base_image_tag_local" \
+--tag "$data_job_base_image_tag_version" \
+--tag "$data_job_base_image_tag_latest" \
+--http-probe=false \
+--exec "/bin/sh -c \"pip3 list && python3 -m pip install --upgrade pip\"" \
+--include-bin "/usr/bin/chmod" \
+--include-bin "/usr/bin/chown" \
+--include-bin "/usr/bin/rm" \
+--include-bin "/usr/bin/bash" \
+--include-bin "/usr/sbin/groupadd" \
+--include-bin "/usr/sbin/groupdel" \
+--include-bin "/usr/sbin/useradd" \
+--include-bin "/usr/sbin/userdel" \
+--include-path "/usr/lib" \
+--include-path "/usr/local/lib/python$PYTHON_MAJOR.$PYTHON_MINOR/" \
+--include-path "/opt/lib/native/oracle"
+
+docker push "$data_job_base_image_tag_version"
+docker push "$data_job_base_image_tag_latest"

projects/control-service/projects/python-image-secure/README.md

@@ -0,0 +1,17 @@
+# Job base image
+
+Job base image is the container "base" image used when building per data job custom image during deployment.
+
+This directory provides the source of some base images for standard python versions.
+It's used by secured installation of VDK.
+
+The current base image installs supporting libraries for some native bindings necessary for installing from source
+some python packages which user may specify for their data job.
+
+## Build
+
+To build the job_base images run `./publish-job-base` which will publish new base image to versatiledatakit container registry.
+
+## Use
+
+It's then set in values.yaml of the helm chart as `deploymentDataJobBaseImage` option

projects/control-service/projects/python-image-secure/publish-python-image.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+VERSION_TAG="${VERSION_TAG:-"0.1dev"}"
+VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"}
+
+PYTHON_MAJOR=$1
+PYTHON_MINOR=$2
+PYTHON_PATCH=$3
+python_name="python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+python_docker_file="Dockerfile-python"
+
+python_image_repo="$VDK_DOCKER_REGISTRY_URL/$python_name"
+python_image_tag_version="$python_image_repo:$VERSION_TAG"
+python_image_tag_latest="$python_image_repo:latest"
+
+docker build -t "$python_image_tag_version" -t "$python_image_tag_latest" -f "$SCRIPT_DIR/$python_docker_file" "$SCRIPT_DIR" \
+--build-arg PYTHON_MAJOR=$PYTHON_MAJOR \
+--build-arg PYTHON_MINOR=$PYTHON_MINOR \
+--build-arg PYTHON_PATCH=$PYTHON_PATCH
+
+docker push "$python_image_tag_version"
+docker push "$python_image_tag_latest"