control-service: split build job base image CI/CD step

projects/control-service/cicd/.gitlab-ci.yml

@@ -145,9 +145,13 @@ control_service_publish_job_base_image:
       changes:
         - projects/control-service/projects/job-base-image/**/*
 
-control_service_publish_job_base_image-secure:
+.control_service_publish_job_base_image_secure:
   extends: .images:dind
   stage: publish_artifacts
+  needs: ["control_service_publish_python_image_secure_3_8",
+    "control_service_publish_python_image_secure_3_9",
+    "control_service_publish_python_image_secure_3_10",
+    "control_service_publish_python_image_secure_3_11"]
   script:
     - apk add --no-cache bash
     - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
@@ -160,14 +164,91 @@ control_service_publish_job_base_image-secure:
         && tar -xvf ds.tar.gz
         && mv dist_linux/docker-slim /usr/local/bin/
         && mv dist_linux/docker-slim-sensor /usr/local/bin/
-    - bash -ex ./publish-job-base.sh
+    - bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR
   retry: !reference [.control_service_retry, retry_options]
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       when: never
     - if: '$CI_COMMIT_BRANCH == "main"'
       changes:
         - projects/control-service/projects/job-base-image-secure/**/*
+        - projects/control-service/projects/python-image-secure/**/*
+
+control_service_publish_job_base_image_secure_3_8:
+  extends: .control_service_publish_job_base_image_secure
+  needs: [ "control_service_publish_python_image_secure_3_8",
+           "control_service_publish_python_image_secure_3_9",
+           "control_service_publish_python_image_secure_3_10",
+           "control_service_publish_python_image_secure_3_11" ]
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 8
+
+control_service_publish_job_base_image_secure_3_10:
+  extends: .control_service_publish_job_base_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 10
+
+control_service_publish_job_base_image_secure_3_11:
+  extends: .control_service_publish_job_base_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 11
+
+.control_service_publish_python_image_secure:
+  extends: .images:dind
+  stage: publish_artifacts
+  script:
+    - apk add --no-cache bash
+    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
+    - cd projects/control-service/projects/python-image-secure
+    - export VERSION_TAG="1.$CI_PIPELINE_ID"
+    - bash -ex ./publish-python-image.sh $PYTHON_MAJOR $PYTHON_MINOR $PYTHON_PATCH
+  retry: !reference [.control_service_retry, retry_options]
+  only:
+    refs:
+      - external_pull_requests
+    changes:
+      - projects/control-service/projects/python-image-secure/**/*
+
+control_service_publish_python_image_secure_3_8:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 8
+    PYTHON_PATCH: 16
+
+control_service_publish_python_image_secure_3_9:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 9
+    PYTHON_PATCH: 16
+
+control_service_publish_python_image_secure_3_10:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 10
+    PYTHON_PATCH: 11
+
+control_service_publish_python_image_secure_3_11:
+  extends: .control_service_publish_python_image_secure
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 11
+    PYTHON_PATCH: 3
+
+control_service_publish_job_base_image_secure_3_9:
+  extends: .control_service_publish_job_base_image_secure
+  needs: [ "control_service_publish_python_image_secure_3_8",
+           "control_service_publish_python_image_secure_3_9",
+           "control_service_publish_python_image_secure_3_10",
+           "control_service_publish_python_image_secure_3_11" ]
+  variables:
+    PYTHON_MAJOR: 3
+    PYTHON_MINOR: 9
 
 control_service_publish_job_builder_image:
   extends: .images:dind

projects/control-service/projects/job-base-image-secure/Dockerfile-data-job-base

@@ -16,3 +16,15 @@ RUN yum erase toybox -y
 
 # Install native dependencies
 RUN yum install shadow build-essential -y
+
+# Install the native dependencies necessary for oracledb python library
+# See https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html
+RUN set -ex \
+      && echo "Installing native dependencies related to support for oracledb python library ..." \
+      && mkdir -p /opt/lib/native  \
+      && yum -y install libaio curl unzip \
+      && curl --insecure --output oracle-instantclient.zip https://download.oracle.com/otn_software/linux/instantclient/2110000/instantclient-basic-linux.x64-21.10.0.0.0dbru.zip \
+      && unzip oracle-instantclient.zip -d /opt/lib/native/oracle && rm -f oracle-instantclient.zip \
+      && sh -c "echo /opt/lib/native/oracle/instantclient_21_10 > /etc/ld.so.conf.d/oracle-instantclient.conf" \
+      && ldconfig \
+      && yum remove -y curl unzip

projects/control-service/projects/job-base-image-secure/README.md

@@ -10,7 +10,7 @@ some python packages which user may specify for their data job.
 
 ## Build
 
-To build the job_base images run `./publish-job-base` which will publish new base image to versatiledatakit container registry.
+To build the job_base images run `./publish-job-base-image` which will publish new base image to versatiledatakit container registry.
 
 ## Use
 

projects/control-service/projects/job-base-image-secure/publish-job-base-image.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+VERSION_TAG="${VERSION_TAG:-"0.1dev"}"
+VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"}
+
+PYTHON_MAJOR=$1
+PYTHON_MINOR=$2
+python_name="python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+data_job_base_name="data-job-base-python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+data_job_base_docker_file="Dockerfile-data-job-base"
+
+python_image_repo="$VDK_DOCKER_REGISTRY_URL/$python_name"
+python_image_tag_latest="$python_image_repo:latest"
+
+data_job_base_image_repo="$VDK_DOCKER_REGISTRY_URL/$data_job_base_name"
+data_job_base_image_tag_local="$data_job_base_image_repo:local"
+data_job_base_image_tag_version="$data_job_base_image_repo:$VERSION_TAG"
+data_job_base_image_tag_latest="$data_job_base_image_repo:latest"
+
+docker build -t "$data_job_base_image_tag_local" -f "$SCRIPT_DIR/$data_job_base_docker_file" "$SCRIPT_DIR" \
+--build-arg base_image="$python_image_tag_latest"
+
+docker-slim build \
+--target "$data_job_base_image_tag_local" \
+--tag "$data_job_base_image_tag_version" \
+--tag "$data_job_base_image_tag_latest" \
+--http-probe=false \
+--exec "/bin/sh -c \"pip3 list && python3 -m pip install --upgrade pip\"" \
+--include-bin "/usr/bin/chmod" \
+--include-bin "/usr/bin/chown" \
+--include-bin "/usr/bin/rm" \
+--include-bin "/usr/bin/bash" \
+--include-bin "/usr/sbin/groupadd" \
+--include-bin "/usr/sbin/groupdel" \
+--include-bin "/usr/sbin/useradd" \
+--include-bin "/usr/sbin/userdel" \
+--include-path "/usr/lib" \
+--include-path "/usr/local/lib/python$PYTHON_MAJOR.$PYTHON_MINOR/" \
+--include-path "/opt/lib/native/oracle"
+
+docker push "$data_job_base_image_tag_version"
+docker push "$data_job_base_image_tag_latest"

projects/control-service/projects/python-image-secure/README.md

@@ -0,0 +1,12 @@
+# Python secure image
+
+The image is based on PhotonOS and contains Python installation.
+
+## Build
+
+To build the Python secure image run `./publish-python-image`
+which will publish new Python to versatiledatakit container registry.
+
+## Use
+
+It is used as base image for `job-base-image-secure`.

projects/control-service/projects/python-image-secure/publish-python-image.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+VERSION_TAG="${VERSION_TAG:-"0.1dev"}"
+VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"}
+
+PYTHON_MAJOR=$1
+PYTHON_MINOR=$2
+PYTHON_PATCH=$3
+python_name="python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+python_docker_file="Dockerfile-python"
+
+python_image_repo="$VDK_DOCKER_REGISTRY_URL/$python_name"
+python_image_tag_version="$python_image_repo:$VERSION_TAG"
+python_image_tag_latest="$python_image_repo:latest"
+
+docker build -t "$python_image_tag_version" -t "$python_image_tag_latest" -f "$SCRIPT_DIR/$python_docker_file" "$SCRIPT_DIR" \
+--build-arg PYTHON_MAJOR=$PYTHON_MAJOR \
+--build-arg PYTHON_MINOR=$PYTHON_MINOR \
+--build-arg PYTHON_PATCH=$PYTHON_PATCH
+
+docker push "$python_image_tag_version"
+docker push "$python_image_tag_latest"