Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

control-service: install necessary dependencies to job builder secure #2472

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions projects/control-service/cicd/.gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,6 @@ control_service_publish_job_base_image:
- docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
- cd projects/control-service/projects/job-base-image-secure
- export VERSION_TAG="1.$CI_PIPELINE_ID"
# Make the docker-slim tool available to the CI job
- >
apk add -u curl
&& curl -L -o ds.tar.gz https://downloads.dockerslim.com/releases/1.37.3/dist_linux.tar.gz
&& tar -xvf ds.tar.gz
&& mv dist_linux/docker-slim /usr/local/bin/
&& mv dist_linux/docker-slim-sensor /usr/local/bin/
- bash -ex ./publish-job-base-image.sh $PYTHON_MAJOR $PYTHON_MINOR
retry: !reference [.control_service_retry, retry_options]
rules:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,3 @@ FROM photon:latest
# Copies essential binaries, libraries, headers, and Python files from the base Python image,
# excluding build dependencies.
COPY --from=base /usr/local/ /usr/local/

# Set the working directory
WORKDIR /job

# Uninstall native dependencies
RUN yum erase toybox -y

# Install native dependencies
RUN yum install shadow build-essential -y

# Install the native dependencies necessary for oracledb python library
# See https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html
RUN set -ex \
&& echo "Installing native dependencies related to support for oracledb python library ..." \
&& mkdir -p /opt/lib/native \
&& yum -y install libaio curl unzip \
&& curl --insecure --output oracle-instantclient.zip https://download.oracle.com/otn_software/linux/instantclient/2110000/instantclient-basic-linux.x64-21.10.0.0.0dbru.zip \
&& unzip oracle-instantclient.zip -d /opt/lib/native/oracle && rm -f oracle-instantclient.zip \
&& sh -c "echo /opt/lib/native/oracle/instantclient_21_10 > /etc/ld.so.conf.d/oracle-instantclient.conf" \
&& ldconfig \
&& yum remove -y curl unzip
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,5 @@ data_job_base_image_tag_latest="$data_job_base_image_repo:latest"
docker build -t "$data_job_base_image_tag_local" -f "$SCRIPT_DIR/$data_job_base_docker_file" "$SCRIPT_DIR" \
--build-arg base_image="$python_image_tag_latest"

docker-slim build \
--target "$data_job_base_image_tag_local" \
--tag "$data_job_base_image_tag_version" \
--tag "$data_job_base_image_tag_latest" \
--http-probe=false \
--exec "/bin/sh -c \"pip3 list && python3 -m pip install --upgrade pip\"" \
--include-bin "/usr/bin/chmod" \
--include-bin "/usr/bin/chown" \
--include-bin "/usr/bin/rm" \
--include-bin "/usr/bin/bash" \
--include-bin "/usr/sbin/groupadd" \
--include-bin "/usr/sbin/groupdel" \
--include-bin "/usr/sbin/useradd" \
--include-bin "/usr/sbin/userdel" \
--include-path "/usr/lib" \
--include-path "/usr/local/lib/python$PYTHON_MAJOR.$PYTHON_MINOR/" \
--include-path "/opt/lib/native/oracle"


docker_push_vdk.sh "$data_job_base_image_tag_version"
docker_push_vdk.sh "$data_job_base_image_tag_latest"
Original file line number Diff line number Diff line change
@@ -1,45 +1,58 @@
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices

ARG base_image=versatiledatakit/data-job-base-python-3.10-secure:latest

FROM $base_image

ARG UID=1000
ARG GID=1000
ARG job_name
ARG job_githash
ARG requirements_file=requirements.txt

ENV JOB_NAME $job_name
ENV VDK_JOB_GITHASH $job_githash
ENV HOME=/job

# Set the working directory
WORKDIR /job

# Validate base image is python based
RUN python -V
# Create necessary users and set home directory to /job
RUN groupadd -r -g $GID group && useradd -u $UID -g $GID -r user && chown -R $UID:$GID /job
ENV HOME=/job

# Copy the actual job that has to be executed
ARG job_name
COPY --chown=$UID:$GID $job_name $job_name/

# Remove execute permissions for files within the Data job directory, but not for the directories themselves
RUN chmod -R -x+X $job_name/*

# TODO: this would trigger for any change in job even if requirements.txt does not change
# but there's no COPY_IF_EXISTS command in docker to try copy it.
ARG requirements_file=requirements.txt
RUN if [ -f "$job_name/$requirements_file" ]; then pip3 install --disable-pip-version-check -q -r "$job_name/$requirements_file" || ( echo ">requirements_failed<" && exit 1 ) ; fi

ARG job_githash
ENV JOB_NAME $job_name
ENV VDK_JOB_GITHASH $job_githash

# Delete system executables
RUN rm /usr/bin/chmod
RUN rm /usr/bin/chown
RUN rm /usr/sbin/groupadd
RUN rm /usr/sbin/groupdel
RUN rm /usr/sbin/useradd
RUN rm /usr/sbin/userdel
RUN rm /usr/bin/uname
RUN python -m pip uninstall pip -y
# Install native dependencies
RUN : \
&& set -ex \
&& echo "Validating base image is python based ..." \
&& python -V \
&& echo "Creating necessary users and set home directory to /job ..." \
&& yum install shadow libffi-devel -y && groupadd -r -g $GID group && useradd -u $UID -g $GID -r user && chown -R $UID:$GID /job && yum autoremove shadow toybox -y \
&& echo "Removing execute permissions for files within the Data job directory, but not for the directories themselves ..." \
&& chmod -R -x+X $job_name/* \
&& if grep -q -E "^oracledb|^cx_Oracle" "$job_name/$requirements_file"; then \
echo "Installing native dependencies related to support for oracledb python library ..." \
&& mkdir -p /opt/lib/native \
&& yum -y install libaio unzip \
&& curl --insecure --output oracle-instantclient.zip https://download.oracle.com/otn_software/linux/instantclient/2110000/instantclient-basic-linux.x64-21.10.0.0.0dbru.zip \
&& unzip oracle-instantclient.zip -d /opt/lib/native/oracle && rm -f oracle-instantclient.zip \
&& sh -c "echo /opt/lib/native/oracle/instantclient_21_10 > /etc/ld.so.conf.d/oracle-instantclient.conf" \
&& ldconfig; fi \
&& if [ -f "$job_name/$requirements_file" ]; then \
echo "Installing native dependencies ..." \
&& yum install shadow build-essential gcc glibc-devel git -y \
&& echo "Installing requirements.txt ..." \
&& pip3 install --disable-pip-version-check -q -r "$job_name/$requirements_file" \
|| ( echo ">requirements_failed<" && exit 1 ) \
&& echo "Removing native dependencies ..." \
&& yum autoremove shadow build-essential gcc glibc-devel git curl unzip -y; fi \
&& echo "Deleting system directories ..." \
&& yum install findutils -y \
&& rm -rf /boot /home /media /mnt /root /sbin /srv /var /usr/lib/ldscripts /usr/lib/rpm /usr/lib/sysimage \
/usr/lib/tdnf /usr/lib/perl5 /usr/lib/gcc /usr/share/locale /tmp/* /usr/include /usr/libexec /usr/sbin /usr/libexec \
&& echo "Deleting system binaries ..." \
&& python -m pip uninstall pip -y \
&& cd /usr/local/bin \
&& ls | grep -xv "python" | grep -xv "python3" | xargs rm -rf \
&& cd /usr/bin \
&& ls | grep -xv "sh" | grep -xv "bash" | xargs rm -rf

USER $UID