control-service: fine-tune the job-builder-secure (#2497)

# Why
We attempted to execute several data jobs utilizing secure images within
our internal deployments. However, we hit a lot of issues:

```
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..data': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/token': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/namespace': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..2023_07_28_14_10_49.411331550/token': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..2023_07_28_14_10_49.411331550/namespace': Read-only file system
rm: cannot remove '/var/run/secrets/kubernetes.io/serviceaccount/..2023_07_28_14_10_49.411331550/ca.crt': Read-only file system
error building image: error building stage: failed to execute command: waiting for process to exit: exit status 1
```

```
Traceback (most recent call last):
  File "/vdk/site-packages/vdk/internal/plugin/plugin.py", line 56, in load_plugins_from_setuptools_entrypoints
    self.__plugin_manager.load_setuptools_entrypoints(self.__group_name)
  File "/vdk/site-packages/pluggy/_manager.py", line 364, in load_setuptools_entrypoints
    plugin = ep.load()
  File "/usr/local/lib/python3.8/importlib/metadata.py", line 77, in load
    module = import_module(match.group('module'))
  File "/usr/local/lib/python3.8/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
  File "<frozen importlib._bootstrap>", line 991, in _find_and_load
  File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 843, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/vdk/site-packages/vdk/plugin/kerberos/kerberos_plugin.py", line 11, in <module>
    from vdk.plugin.kerberos.authenticator_factory import KerberosAuthenticatorFactory
  File "/vdk/site-packages/vdk/plugin/kerberos/authenticator_factory.py", line 10, in <module>
    from vdk.plugin.kerberos.minikerberos_authenticator import (
  File "/vdk/site-packages/vdk/plugin/kerberos/minikerberos_authenticator.py", line 8, in <module>
    from minikerberos.common.creds import KerberosCredential
  File "/vdk/site-packages/minikerberos/common/creds.py", line 32, in <module>
    from oscrypto.asymmetric import rsa_pkcs1v15_sign, load_private_key
  File "/vdk/site-packages/oscrypto/asymmetric.py", line 19, in <module>
    from ._asymmetric import _unwrap_private_key_info
  File "/vdk/site-packages/oscrypto/_asymmetric.py", line 27, in <module>
    from .kdf import pbkdf1, pbkdf2, pkcs12_kdf
  File "/vdk/site-packages/oscrypto/kdf.py", line 9, in <module>
    from .util import rand_bytes
  File "/vdk/site-packages/oscrypto/util.py", line 14, in <module>
    from ._openssl.util import rand_bytes
  File "/vdk/site-packages/oscrypto/_openssl/util.py", line 6, in <module>
    from ._libcrypto import libcrypto, libcrypto_version_info, handle_openssl_error
  File "/vdk/site-packages/oscrypto/_openssl/_libcrypto.py", line 9, in <module>
    from ._libcrypto_cffi import (
  File "/vdk/site-packages/oscrypto/_openssl/_libcrypto_cffi.py", line 27, in <module>
    raise LibraryNotFoundError('The library libcrypto could not be found')
oscrypto.errors.LibraryNotFoundError: The library libcrypto could not be found
warning: Plugin load failed
```

# What
We have made updates to the native dependencies of the image and
reverted to the base job image model.

# Testing Done
Execution of data jobs within the internal deployment environment.

Signed-off-by: Miroslav Ivanov [email protected]

---------

Signed-off-by: Miroslav Ivanov [email protected]

projects/control-service/cicd/.gitlab-ci.yml

@@ -162,7 +162,7 @@ control_service_publish_job_base_image:
     - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
     - cd projects/control-service/projects/job-base-image-secure
     - export VERSION_TAG="1.$CI_PIPELINE_ID"
-    - bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR
+    - bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR $PYTHON_PATCH
   retry: !reference [.control_service_retry, retry_options]
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
@@ -177,68 +177,27 @@ control_service_publish_job_base_image_secure_3_8:
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 8
+    PYTHON_PATCH: 16
 
 control_service_publish_job_base_image_secure_3_9:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 9
+    PYTHON_PATCH: 16
 
 control_service_publish_job_base_image_secure_3_10:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 10
+    PYTHON_PATCH: 11
 
 control_service_publish_job_base_image_secure_3_11:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 11
-
-.control_service_publish_python_image_secure:
-  extends: .images:dind
-  stage: publish_artifacts
-  script:
-    - apk add --no-cache bash
-    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
-    - cd projects/control-service/projects/python-image-secure
-    - export VERSION_TAG="1.$CI_PIPELINE_ID"
-    - bash -ex ./publish-python-image.sh $PYTHON_MAJOR $PYTHON_MINOR $PYTHON_PATCH
-  retry: !reference [.control_service_retry, retry_options]
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      when: never
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes:
-        - projects/control-service/projects/python-image-secure/**/*
-
-control_service_publish_python_image_secure_3_8:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 8
-    PYTHON_PATCH: 16
-
-control_service_publish_python_image_secure_3_9:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 9
-    PYTHON_PATCH: 16
-
-control_service_publish_python_image_secure_3_10:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 10
-    PYTHON_PATCH: 11
-
-control_service_publish_python_image_secure_3_11:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 11
     PYTHON_PATCH: 3
 
 control_service_publish_job_builder_image:

projects/control-service/projects/job-base-image-secure/Dockerfile-data-job-base

@@ -1,9 +1,98 @@
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices
-ARG base_image
-FROM $base_image as base
+FROM photon:latest as build
+
+ARG PYTHON_MAJOR
+ARG PYTHON_MINOR
+ARG PYTHON_PATCH
+ARG PYTHON_VERSION=${PYTHON_MAJOR}.${PYTHON_MINOR}.${PYTHON_PATCH}
+
+ARG _prefixdir=/usr/local
+ARG _bindir=${_prefixdir}/bin
+ARG _libdir=${_prefixdir}/lib
+ARG _workdir=/usr/src
+ARG _pylibdir=${_libdir}/python${PYTHON_MAJOR}.${PYTHON_MINOR}
+ARG _bytecode_suffixes=.cpython-*.pyc
+
+ENV PATH=${_bindir}:${PATH}
+WORKDIR ${_workdir}
+
+# Install build dependencies
+RUN yum install -y \
+    coreutils \
+    gcc \
+    glibc-devel \
+    binutils \
+    build-essential \
+    wget \
+    make \
+    openssl-devel \
+    bzip2-devel \
+    libffi-devel \
+    zlib-devel \
+    sqlite-devel \
+    krb5-devel \
+    e2fsprogs-devel
+
+# Extract python source
+RUN : \
+ && set -ex \
+ && curl -O https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
+ && tar -xvzf Python-${PYTHON_VERSION}.tgz \
+ && mv Python-${PYTHON_VERSION} python \
+ && rm Python-${PYTHON_VERSION}.tgz
+
+# Build and install python
+RUN : \
+ && set -ex \
+ && cd ${_workdir}/python \
+ && ./configure ax_cv_c_float_words_bigendian=no \
+        --enable-loadable-sqlite-extensions \
+        --enable-optimizations \
+        --enable-option-checking=fatal \
+        --enable-shared \
+        --with-lto \
+        --without-ensurepip \
+        --prefix=${_prefixdir} \
+        LDFLAGS=-Wl,-rpath=${_libdir} \
+ && make \
+ && make install
+
+# Make some useful symlinks
+RUN : \
+ && set -ex \
+ && cd ${_bindir} \
+ && ln -s python${PYTHON_MAJOR} python
+
+# Get and install pip
+RUN : \
+ && set -ex \
+ && curl -O https://bootstrap.pypa.io/get-pip.py \
+ && python get-pip.py \
+ && pip --version \
+ && rm -f get-pip.py
+
+# Cleanup files
+RUN : \
+ && set -ex \
+ && rm -rf \
+      ${_workdir}/python \
+      ${_pylibdir}/turtle.py \
+      ${_pylibdir}/__pycache__/turtle*${_bytecode_suffixes} \
+      ${_bindir}/idle* \
+      ${_pylibdir}/idlelib \
+      ${_pylibdir}/tkinter \
+      ${_pylibdir}/turtledemo \
+      ${_pylibdir}/ctypes/test \
+      ${_pylibdir}/distutils/tests \
+      ${_pylibdir}/lib2to3/tests \
+      ${_pylibdir}/sqlite3/test \
+      ${_pylibdir}/test \
+      ${_pylibdir}/tkinter/test \
+      ${_pylibdir}/unittest/test \
+ && find ${_pylibdir} -type d -name __pycache__ -exec rm -rf '{}' +
 
 FROM photon:latest
 
 # Copies essential binaries, libraries, headers, and Python files from the base Python image,
 # excluding build dependencies.
-COPY --from=base /usr/local/ /usr/local/
+COPY --from=build /usr/local/ /usr/local/

projects/control-service/projects/job-base-image-secure/publish-job-base-image.sh

@@ -9,20 +9,19 @@ VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/vers
 
 PYTHON_MAJOR=$1
 PYTHON_MINOR=$2
-python_name="python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+PYTHON_PATCH=$3
 data_job_base_name="data-job-base-python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
 data_job_base_docker_file="Dockerfile-data-job-base"
 
-python_image_repo="$VDK_DOCKER_REGISTRY_URL/$python_name"
-python_image_tag_latest="$python_image_repo:latest"
-
 data_job_base_image_repo="$VDK_DOCKER_REGISTRY_URL/$data_job_base_name"
 data_job_base_image_tag_version="$data_job_base_image_repo:$VERSION_TAG"
 data_job_base_image_tag_latest="$data_job_base_image_repo:latest"
 
 docker build -t "$data_job_base_image_tag_version" -t "$data_job_base_image_tag_latest" \
 -f "$SCRIPT_DIR/$data_job_base_docker_file" "$SCRIPT_DIR" \
---build-arg base_image="$python_image_tag_latest"
+--build-arg PYTHON_MAJOR=$PYTHON_MAJOR \
+--build-arg PYTHON_MINOR=$PYTHON_MINOR \
+--build-arg PYTHON_PATCH=$PYTHON_PATCH
 
 docker_push_vdk.sh "$data_job_base_image_tag_version"
 docker_push_vdk.sh "$data_job_base_image_tag_latest"

projects/control-service/projects/job-builder-secure/Dockerfile.python.vdk

@@ -25,7 +25,9 @@ RUN : \
      && echo "Validating base image is python based ..." \
      && python -V \
      && echo "Creating necessary users and set home directory to /job ..." \
-     && yum install shadow libffi-devel -y && groupadd -r -g $GID group && useradd -u $UID -g $GID -r user && chown -R $UID:$GID /job && yum autoremove shadow toybox -y \
+     && yum install shadow -y \
+     && groupadd -r -g $GID group && useradd -u $UID -g $GID -r user \
+     && chown -R $UID:$GID /job \
      && echo "Removing execute permissions for files within the Data job directory, but not for the directories themselves ..." \
      && chmod -R -x+X $job_name/* \
      && if grep -q -E "^oracledb|^cx_Oracle" "$job_name/$requirements_file"; then \
@@ -38,20 +40,27 @@ RUN : \
            && ldconfig; fi \
     && if [ -f "$job_name/$requirements_file" ]; then \
           echo "Installing native dependencies ..." \
-          && yum install shadow build-essential gcc glibc-devel git -y \
+          && yum install build-essential gcc glibc-devel git -y \
           && echo "Installing requirements.txt ..." \
-          && pip3 install --disable-pip-version-check -q -r "$job_name/$requirements_file"  \
+          && pip install --disable-pip-version-check -q -r "$job_name/$requirements_file"  \
           || ( echo ">requirements_failed<" && exit 1 ) \
           && echo "Removing native dependencies ..." \
-          && yum autoremove shadow build-essential gcc glibc-devel git curl unzip -y;  fi \
+          && yum autoremove build-essential gcc glibc-devel git unzip -y;  fi \
+    && echo "Installing native dependencies ..." \
+    && yum install libffi-devel libstdc++ findutils openssl-c_rehash -y \
+    && echo "Refreshing CA certificates ..." \
+    && /usr/bin/rehash_ca_certificates.sh \
+    && echo "Deleting system packages ..." \
+    && yum autoremove shadow toybox openssl-c_rehash -y \
     && echo "Deleting system directories ..." \
-    && yum install findutils -y  \
-    && rm -rf /boot /home /media /mnt /root /sbin /srv /var /usr/lib/ldscripts /usr/lib/rpm /usr/lib/sysimage  \
-    /usr/lib/tdnf /usr/lib/perl5 /usr/lib/gcc /usr/share/locale /tmp/* /usr/include /usr/libexec /usr/sbin /usr/libexec  \
+    && rm -rf /boot /home /media /mnt /root /srv /usr/lib/ldscripts /usr/lib/rpm /usr/lib/sysimage \
+    /usr/lib/tdnf /usr/lib/perl5 /usr/lib/gcc /usr/share/locale /tmp/* /usr/include /usr/libexec /usr/libexec  \
     && echo "Deleting system binaries ..." \
     && python -m pip uninstall pip -y \
+    && cd /usr/sbin \
+    && ls | grep -xv "ldconfig" | xargs rm -rf \
     && cd /usr/local/bin \
-    && ls | grep -xv "python" | grep -xv "python3" | xargs rm -rf \
+    && ls | grep -xv "python" | grep -xv "python3" | grep -xv $(python -c 'import sys; print("python"+str(sys.version_info[0])+"."+str(sys.version_info[1]))') | xargs rm -rf \
     && cd /usr/bin \
     && ls | grep -xv "sh" | grep -xv "bash" | xargs rm -rf
 

projects/control-service/projects/job-builder-secure/version.txt

@@ -1 +1 @@
-1.3.0
+1.3.1