Merge pull request #370 from taharah/multi-stage-builds

Update all Dockerfiles to use multi-stage builds

Dockerfile

@@ -1,31 +1,53 @@
-FROM python:3.7-slim-stretch
+# First build the Helm binding
+FROM golang:1.12.9-stretch AS helm-builder
 
 RUN mkdir /kapitan
 WORKDIR /kapitan
-COPY kapitan/ kapitan/
-COPY requirements.txt ./
 
-RUN apt-get update && apt-get install -y \
-    build-essential git wget \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
+COPY ./kapitan ./kapitan
+COPY ./MANIFEST.in ./MANIFEST.in
+COPY ./requirements.txt ./requirements.txt
+COPY ./setup.py ./setup.py
+
+RUN chmod +x ./kapitan/inputs/helm/build.sh \
+    && ./kapitan/inputs/helm/build.sh
+
+# Build the virtualenv for Kapitan
+FROM python:3.7-slim-stretch AS python-builder
+
+COPY --from=helm-builder /kapitan /kapitan
+WORKDIR /kapitan
+
+ENV PATH="/opt/venv/bin:${PATH}"
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        build-essential \
+    && python -m venv /opt/venv \
     && pip install --upgrade pip \
-    && pip install -r requirements.txt
+    && pip install -r requirements.txt \
+    && ./kapitan/inputs/helm/build.sh \
+    && pip install .
 
-# build go from source
-RUN wget https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz -q && \
-    tar -C /usr/local -xvf go1.12.7.linux-amd64.tar.gz && \
-    rm go1.12.7.linux-amd64.tar.gz
+# Final image with virtualenv built in previous step
+FROM python:3.7-slim-stretch
 
-# build helm binding
-RUN export PATH=$PATH:/usr/local/go/bin && \
-    chmod +x kapitan/inputs/helm/build.sh && \
-    ./kapitan/inputs/helm/build.sh && \
-    rm /usr/local/go -rf
+COPY --from=python-builder /opt/venv /opt/venv
 
-ENV PYTHONPATH="/kapitan/"
+ENV PATH="/opt/venv/bin:${PATH}"
 ENV SEARCHPATH="/src"
 VOLUME ${SEARCHPATH}
 WORKDIR ${SEARCHPATH}
 
-ENTRYPOINT ["python", "-m", "kapitan"]
+# Install runtime dependencies and run as a non-root user for good measure
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        git \
+        gnupg \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd --create-home --no-log-init --user-group kapitan
+
+USER kapitan
+
+ENTRYPOINT ["kapitan"]

Dockerfile.ci

@@ -1,57 +1,78 @@
-FROM golang:1.12-stretch AS go-env
+# /bin/terraform
+FROM hashicorp/terraform:0.12.8 AS terraform-binary
 
-RUN go get -u github.com/prometheus/prometheus/cmd/promtool
+# /bin/promtool
+FROM quay.io/prometheus/prometheus:v2.12.0 AS prometheus-binary
 
-ARG inputs_path=kapitan/inputs
-COPY $inputs_path/helm/template.go $inputs_path/helm/go.mod $inputs_path/helm/build.sh /kapitan/helm/
+# Build the Helm binding
+FROM golang:1.12-stretch AS helm-builder
 
-RUN /kapitan/helm/build.sh
+RUN mkdir /kapitan
+WORKDIR /kapitan
 
+COPY ./kapitan ./kapitan
+COPY ./MANIFEST.in ./MANIFEST.in
+COPY ./requirements.txt ./requirements.txt
+COPY ./setup.py ./setup.py
+
+RUN chmod +x ./kapitan/inputs/helm/build.sh \
+    && ./kapitan/inputs/helm/build.sh
+
+# Build final image
 FROM python:3.7-stretch
 
-RUN apt-get update && apt-get install -y \
-    build-essential git curl jq libncursesw5-dev libffi-dev zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev \
-    libssl-dev libreadline-dev libffi-dev wget libbz2-dev libsqlite3-dev bash zip lsb-release && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+ARG CLOUD_SDK_VERSION=262.0.0
 
-ARG CLOUD_SDK_VERSION=260.0.0
 ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION
+ENV KAPP_URL=https://github.com/k14s/kapp/releases/download/v0.11.0/kapp-linux-amd64
+ENV KBLD_URL=https://github.com/k14s/kbld/releases/download/v0.11.0/kbld-linux-amd64
+ENV PATH="/opt/venv/bin:${PATH}"
 
-ENV PATH "$PATH:/opt/google-cloud-sdk/bin/"
+COPY --from=terraform-binary /bin/terraform /usr/bin/terraform
+COPY --from=prometheus-binary /bin/promtool /user/bin/promtool
+COPY --from=helm-builder /kapitan /kapitan
 
-RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
-    mkdir -p /opt && cd /opt && \
-    curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
-    tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
-    rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
-    gcloud config set core/disable_usage_reporting true && \
-    gcloud config set component_manager/disable_update_check true && \
-    gcloud config set metrics/environment github_docker_image && \
-    gcloud --version
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        apt-transport-https \
+        bash \
+        build-essential \
+        curl \
+        git \
+        jq \
+        lsb-release \
+        gnupg \
+        wget \
+        zip \
+    && export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \
+    && echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list \
+    && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+    && apt-get update \
+    && apt-get install -y \
+        google-cloud-sdk=${CLOUD_SDK_VERSION}-0 \
+        kubectl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    # If True, anonymous statistics on SDK usage will not be collected.
+    && gcloud config set core/disable_usage_reporting true \
+    # If True, Cloud SDK will not automatically check for updates.
+    && gcloud config set component_manager/disable_update_check true \
+    # Metrics namespace for server side analytics
+    && gcloud config set metrics/environment github_docker_image \
+    && python -m venv /opt/venv \
+    && pip install --upgrade pip \
+    && pip install -r ./kapitan/requirements.txt \
+    && ./kapitan/kapitan/inputs/helm/build.sh \
+    && pip install ./kapitan \
+    && rm -rf ./kapitan \
+    && curl -L -o /usr/local/bin/kapp ${KAPP_URL} \
+    && chmod +x /usr/local/bin/kapp \
+    && curl -L -o /usr/local/bin/kbld ${KBLD_URL} \
+    && chmod +x /usr/local/bin/kbld \
+    && gcloud --version \
+    && kubectl version --client \
+    && terraform --version
 
 VOLUME ["/root/.config"]
 
-COPY setup.py requirements.txt MANIFEST.in /kapitan/
-COPY kapitan /kapitan/kapitan/
-# add helm binding to kapitan/kapitan before pip install
-COPY --from=go-env /kapitan/helm/libtemplate.so /kapitan/kapitan/inputs/helm/
-RUN pip install cffi && python /kapitan/kapitan/inputs/helm/cffi_build.py && mv ./helm_binding.py /kapitan/kapitan/inputs/helm/
-RUN pip install -e /kapitan
-
-ENV TERRAFORM_VERSION=0.11.13
-ENV TERRAFORM_URL=https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip
-RUN curl -o /tmp/terraform_${TERRAFORM_VERSION}.zip ${TERRAFORM_URL} && \
-        unzip -d /usr/local/bin /tmp/terraform_${TERRAFORM_VERSION}.zip && \
-        rm /tmp/terraform_${TERRAFORM_VERSION}.zip
-
-RUN gcloud components install kubectl
-
-ENV KAPP_URL=https://github.com/k14s/kapp/releases/download/v0.11.0/kapp-linux-amd64
-ENV KBLD_URL=https://github.com/k14s/kbld/releases/download/v0.11.0/kbld-linux-amd64
-RUN curl -L -o /usr/local/bin/kapp ${KAPP_URL} && chmod +x /usr/local/bin/kapp
-RUN curl -L -o /usr/local/bin/kbld ${KBLD_URL} && chmod +x /usr/local/bin/kbld
-
-COPY --from=go-env /go/bin/promtool /usr/local/bin/
-
 CMD ["/bin/bash"]

Dockerfile.pyinstaller

@@ -1,40 +1,37 @@
-FROM debian:stretch
+# First build the Helm binding
+FROM golang:1.12.9-stretch AS helm-builder
+
+RUN mkdir /kapitan
+WORKDIR /kapitan
+
+COPY ./kapitan ./kapitan
+COPY ./MANIFEST.in ./MANIFEST.in
+COPY ./requirements.txt ./requirements.txt
+COPY ./scripts/pyinstaller.sh ./pyinstaller.sh
+COPY ./setup.py ./setup.py
+
+RUN chmod +x ./kapitan/inputs/helm/build.sh \
+    && ./kapitan/inputs/helm/build.sh
+
+FROM python:3.7-slim-stretch
+
 SHELL ["/bin/bash", "-i", "-c"]
 
-ARG PYTHON_VERSION=3.7.2
 ARG PYINSTALLER_VERSION=3.5
 
-RUN mkdir /kapitan
+COPY --from=helm-builder /kapitan /kapitan
 WORKDIR /kapitan
-COPY kapitan/ kapitan/
-# COPY __main__.spec ./build.spec
-COPY requirements.txt ./requirements.txt
-COPY scripts/pyinstaller.sh ./pyinstaller.sh
+
+ENV PATH="/opt/venv/bin:${PATH}"
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    git \
-    libffi-dev \
-    libbz2-dev \
-    libreadline-dev \
-    libsqlite3-dev \
-    libssl-dev \
-    zlib1g-dev \
-    curl
-
-RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc \
-    && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc \
-    && source ~/.bashrc \
-    && curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \
-    && echo 'eval "$(pyenv init -)"' >> ~/.bashrc \
-    && source ~/.bashrc \
-    && PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYTHON_VERSION \
-    && pyenv global $PYTHON_VERSION \
+    && apt-get install --no-install-recommends -y \
+        build-essential \
+    && python -m venv /opt/venv \
     && pip install --upgrade pip \
     && pip install pyinstaller==$PYINSTALLER_VERSION \
     && pip install -r requirements.txt \
-    && chmod +x pyinstaller.sh
+    && ./kapitan/inputs/helm/build.sh \
+    && chmod +x ./pyinstaller.sh
 
 ENTRYPOINT ["./pyinstaller.sh"]

kapitan/inputs/helm/build.sh

@@ -14,7 +14,7 @@ fi
 
 # Validate that the compiled binding exists
 if [[ -e $so_name ]]; then
-    echo "$so_name built successfully"
+    echo "[INFO] $so_name built successfully or already exists"
 else
     echo "[ERROR] $so_name does not exist!"
     exit 1
@@ -24,7 +24,7 @@ fi
 if [[ -z $(which python3) ]]; then
     echo "[WARN] python3 is not available on this system -- skipping cffi build!"
 else
-    echo "Building the Python binding using cffi"
+    echo "[INFO] Building the Python binding using cffi"
     python3 cffi_build.py
 fi
 

scripts/pyinstaller.sh

@@ -4,7 +4,6 @@
 # created by Dockerfile.pyinstaller
 
 set -e
-. /root/.bashrc
 
 entry='__main__'
 output_name='kapitan-linux-amd64'
@@ -19,4 +18,4 @@ mv dist/$entry dist/$output_name
 # Open permissions so that when this binary
 # is used outside of docker (on the volume mount) it
 # also can be deleted by Travis CI
-chmod 777 dist/$output_name
+chmod 777 dist/$output_name