[DRAFT] control-service: fine-tune the job-builder-secure

Signed-off-by: Miroslav Ivanov [email protected]

projects/control-service/cicd/.gitlab-ci.yml

@@ -89,80 +89,38 @@ control_service_publish_job_base_image:
     - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
     - cd projects/control-service/projects/job-base-image-secure
     - export VERSION_TAG="1.$CI_PIPELINE_ID"
-    - bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR
+    - bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR $PYTHON_PATCH
   retry: !reference [.control_service_retry, retry_options]
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      when: never
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      changes:
-        - projects/control-service/projects/job-base-image-secure/**/*
-        - projects/control-service/projects/python-image-secure/**/*
+  only:
+    refs:
+      - external_pull_requests
 
 control_service_publish_job_base_image_secure_3_8:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 8
+    PYTHON_PATCH: 16
 
 control_service_publish_job_base_image_secure_3_9:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 9
+    PYTHON_PATCH: 16
 
 control_service_publish_job_base_image_secure_3_10:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 10
+    PYTHON_PATCH: 11
 
 control_service_publish_job_base_image_secure_3_11:
   extends: .control_service_publish_job_base_image_secure
   variables:
     PYTHON_MAJOR: 3
     PYTHON_MINOR: 11
-
-.control_service_publish_python_image_secure:
-  extends: .images:dind
-  stage: publish_artifacts
-  script:
-    - apk add --no-cache bash
-    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
-    - cd projects/control-service/projects/python-image-secure
-    - export VERSION_TAG="1.$CI_PIPELINE_ID"
-    - bash -ex ./publish-python-image.sh $PYTHON_MAJOR $PYTHON_MINOR $PYTHON_PATCH
-  retry: !reference [.control_service_retry, retry_options]
-  only:
-    refs:
-      - external_pull_requests
-
-control_service_publish_python_image_secure_3_8:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 8
-    PYTHON_PATCH: 16
-
-control_service_publish_python_image_secure_3_9:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 9
-    PYTHON_PATCH: 16
-
-control_service_publish_python_image_secure_3_10:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 10
-    PYTHON_PATCH: 11
-
-control_service_publish_python_image_secure_3_11:
-  extends: .control_service_publish_python_image_secure
-  variables:
-    PYTHON_MAJOR: 3
-    PYTHON_MINOR: 11
     PYTHON_PATCH: 3
 
 control_service_publish_job_builder_secure_image:

projects/control-service/projects/job-base-image-secure/Dockerfile-data-job-base

@@ -1,9 +1,98 @@
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices
-ARG base_image
-FROM $base_image as base
+FROM photon:latest as build
+
+ARG PYTHON_MAJOR
+ARG PYTHON_MINOR
+ARG PYTHON_PATCH
+ARG PYTHON_VERSION=${PYTHON_MAJOR}.${PYTHON_MINOR}.${PYTHON_PATCH}
+
+ARG _prefixdir=/usr/local
+ARG _bindir=${_prefixdir}/bin
+ARG _libdir=${_prefixdir}/lib
+ARG _workdir=/usr/src
+ARG _pylibdir=${_libdir}/python${PYTHON_MAJOR}.${PYTHON_MINOR}
+ARG _bytecode_suffixes=.cpython-*.pyc
+
+ENV PATH=${_bindir}:${PATH}
+WORKDIR ${_workdir}
+
+# Install build dependencies
+RUN yum install -y \
+    coreutils \
+    gcc \
+    glibc-devel \
+    binutils \
+    build-essential \
+    wget \
+    make \
+    openssl-devel \
+    bzip2-devel \
+    libffi-devel \
+    zlib-devel \
+    sqlite-devel \
+    krb5-devel \
+    e2fsprogs-devel
+
+# Extract python source
+RUN : \
+ && set -ex \
+ && curl -O https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
+ && tar -xvzf Python-${PYTHON_VERSION}.tgz \
+ && mv Python-${PYTHON_VERSION} python \
+ && rm Python-${PYTHON_VERSION}.tgz
+
+# Build and install python
+RUN : \
+ && set -ex \
+ && cd ${_workdir}/python \
+ && ./configure ax_cv_c_float_words_bigendian=no \
+        --enable-loadable-sqlite-extensions \
+        --enable-optimizations \
+        --enable-option-checking=fatal \
+        --enable-shared \
+        --with-lto \
+        --without-ensurepip \
+        --prefix=${_prefixdir} \
+        LDFLAGS=-Wl,-rpath=${_libdir} \
+ && make \
+ && make install
+
+# Make some useful symlinks
+RUN : \
+ && set -ex \
+ && cd ${_bindir} \
+ && ln -s python${PYTHON_MAJOR} python
+
+# Get and install pip
+RUN : \
+ && set -ex \
+ && curl -O https://bootstrap.pypa.io/get-pip.py \
+ && python get-pip.py \
+ && pip --version \
+ && rm -f get-pip.py
+
+# Cleanup files
+RUN : \
+ && set -ex \
+ && rm -rf \
+      ${_workdir}/python \
+      ${_pylibdir}/turtle.py \
+      ${_pylibdir}/__pycache__/turtle*${_bytecode_suffixes} \
+      ${_bindir}/idle* \
+      ${_pylibdir}/idlelib \
+      ${_pylibdir}/tkinter \
+      ${_pylibdir}/turtledemo \
+      ${_pylibdir}/ctypes/test \
+      ${_pylibdir}/distutils/tests \
+      ${_pylibdir}/lib2to3/tests \
+      ${_pylibdir}/sqlite3/test \
+      ${_pylibdir}/test \
+      ${_pylibdir}/tkinter/test \
+      ${_pylibdir}/unittest/test \
+ && find ${_pylibdir} -type d -name __pycache__ -exec rm -rf '{}' +
 
 FROM photon:latest
 
-# Copies essential binaries, libraries, headers, and Python files from the base Python image,
-# excluding build dependencies.
-COPY --from=base /usr/local/ /usr/local/
+ # Copies essential binaries, libraries, headers, and Python files from the base Python image,
+ # excluding build dependencies.
+COPY --from=build /usr/local/ /usr/local/

projects/control-service/projects/job-base-image-secure/publish-job-base-image.sh

@@ -9,20 +9,19 @@ VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/vers
 
 PYTHON_MAJOR=$1
 PYTHON_MINOR=$2
-python_name="python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
+PYTHON_PATCH=$3
 data_job_base_name="data-job-base-python-$PYTHON_MAJOR.$PYTHON_MINOR-secure"
 data_job_base_docker_file="Dockerfile-data-job-base"
 
-python_image_repo="$VDK_DOCKER_REGISTRY_URL/$python_name"
-python_image_tag_latest="$python_image_repo:latest"
-
 data_job_base_image_repo="$VDK_DOCKER_REGISTRY_URL/$data_job_base_name"
 data_job_base_image_tag_version="$data_job_base_image_repo:$VERSION_TAG"
 data_job_base_image_tag_latest="$data_job_base_image_repo:latest"
 
 docker build -t "$data_job_base_image_tag_version" -t "$data_job_base_image_tag_latest" \
 -f "$SCRIPT_DIR/$data_job_base_docker_file" "$SCRIPT_DIR" \
---build-arg base_image="$python_image_tag_latest"
+--build-arg PYTHON_MAJOR=$PYTHON_MAJOR \
+--build-arg PYTHON_MINOR=$PYTHON_MINOR \
+--build-arg PYTHON_PATCH=$PYTHON_PATCH
 
 docker_push_vdk.sh "$data_job_base_image_tag_version"
 docker_push_vdk.sh "$data_job_base_image_tag_latest"

projects/control-service/projects/job-builder-secure/Dockerfile.python.vdk

@@ -1,13 +1,7 @@
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices
 
 ARG base_image=versatiledatakit/data-job-base-python-3.10-secure:latest
-FROM $base_image as base
-
-FROM photon:latest
-
-# Copies essential binaries, libraries, headers, and Python files from the base Python image,
-# excluding build dependencies.
-COPY --from=base /usr/local/ /usr/local/
+FROM $base_image
 
 ARG UID=1000
 ARG GID=1000
@@ -31,7 +25,9 @@ RUN : \
      && echo "Validating base image is python based ..." \
      && python -V \
      && echo "Creating necessary users and set home directory to /job ..." \
-     && yum install shadow openssl-c_rehash -y && /usr/bin/rehash_ca_certificates.sh && groupadd -r -g $GID group && useradd -u $UID -g $GID -r user && chown -R $UID:$GID /job && yum autoremove shadow toybox openssl-c_rehash -y \
+     && yum install shadow -y \
+     && groupadd -r -g $GID group && useradd -u $UID -g $GID -r user \
+     && chown -R $UID:$GID /job \
      && echo "Removing execute permissions for files within the Data job directory, but not for the directories themselves ..." \
      && chmod -R -x+X $job_name/* \
      && if grep -q -E "^oracledb|^cx_Oracle" "$job_name/$requirements_file"; then \
@@ -44,14 +40,19 @@ RUN : \
            && ldconfig; fi \
     && if [ -f "$job_name/$requirements_file" ]; then \
           echo "Installing native dependencies ..." \
-          && yum install shadow build-essential gcc glibc-devel git -y \
+          && yum install build-essential gcc glibc-devel git -y \
           && echo "Installing requirements.txt ..." \
           && pip install --disable-pip-version-check -q -r "$job_name/$requirements_file"  \
           || ( echo ">requirements_failed<" && exit 1 ) \
           && echo "Removing native dependencies ..." \
-          && yum autoremove shadow build-essential gcc glibc-devel git unzip -y;  fi \
+          && yum autoremove build-essential gcc glibc-devel git unzip -y;  fi \
+    && echo "Installing native dependencies ..." \
+    && yum install libffi-devel libstdc++ findutils openssl-c_rehash -y \
+    && echo "Refreshing CA certificates ..." \
+    && /usr/bin/rehash_ca_certificates.sh \
+    && echo "Deleting system packages ..." \
+    && yum autoremove shadow toybox openssl-c_rehash -y \
     && echo "Deleting system directories ..." \
-    && yum install libffi-devel libstdc++ findutils -y \
     && rm -rf /boot /home /media /mnt /root /srv /usr/lib/ldscripts /usr/lib/rpm /usr/lib/sysimage \
     /usr/lib/tdnf /usr/lib/perl5 /usr/lib/gcc /usr/share/locale /tmp/* /usr/include /usr/libexec /usr/libexec  \
     && echo "Deleting system binaries ..." \
@@ -60,6 +61,7 @@ RUN : \
     && ls | grep -xv "ldconfig" | xargs rm -rf \
     && cd /usr/local/bin \
     && ls | grep -xv "python" | grep -xv "python3" | grep -xv $(python -c 'import sys; print("python"+str(sys.version_info[0])+"."+str(sys.version_info[1]))') | xargs rm -rf \
+    && ls /etc/ssl/certs \
     && cd /usr/bin \
     && ls | grep -xv "sh" | grep -xv "bash" | xargs rm -rf