Create custom image that does not have ssh keys stored (#343)

This removes most keys from all images.

The operator requires the dbadmin and root account to have valid keys. This is
needed to support a cluster with mixed releases as admintools uses ssh to
communicate between hosts. So, they must be preserved and kept static.

We offer a new version of the image with the dbadmin and root account keys
gone. This can be built with a new variable (NO_SSH_KEYS). When using this
version of the image, you must add your own ssh keys into a secret and provide
it in the VerticaDB spec.sshSecret field.

The GitHub action (e2e-leg-1) have been modified to use that custom image.

.github/actions/setup-e2e/action.yaml

@@ -26,6 +26,7 @@ runs:
       run: |
         find . -name \*.tar
         docker load --input full-vertica-image/full-vertica-image.tar
+        docker load --input nosshkeys-vertica-image/nosshkeys-vertica-image.tar
         docker load --input minimal-vertica-image/minimal-vertica-image.tar
         docker load --input operator-image/operator-image.tar
         docker load --input vlogger-image/vlogger-image.tar

.github/workflows/build-images.yml

@@ -11,6 +11,10 @@ on:
         description: 'Name of an existing full Vertica server image. If blank we will build one using the default name'
         type: string
         required: true
+      nosshkeys_vertica_image:
+        description: 'Name of an existing Vertica server image with no ssh keys inside. If blank we will build one using the default name'
+        type: string
+        required: true
       minimal_vertica_image:
         description: 'Name of an existing minimal Vertica server image. If blank we will build one using the default name'
         type: string
@@ -43,6 +47,9 @@ on:
       full-vertica-image:
         description: "The image name of the full vertica server image"
         value: ${{ jobs.build-server-full.outputs.image }}
+      nosshkeys-vertica-image:
+        description: "The image name of the vertica server image, but with no ssh keys inside"
+        value: ${{ jobs.build-server-nosshkeys.outputs.image }}
       minimal-vertica-image:
         description: "The image name of the vertica server, but with optional software removed"
         value: ${{ jobs.build-server-minimal.outputs.image }}
@@ -178,6 +185,89 @@ jobs:
         echo -n "Vertica Version: **" >> $GITHUB_STEP_SUMMARY
         echo -n $(docker inspect --format '{{index .Config.Labels "vertica-version"}}' ${{ steps.full_vertica_image.outputs.value }}) >> $GITHUB_STEP_SUMMARY
         echo "**" >> $GITHUB_STEP_SUMMARY
+        
+  build-server-nosshkeys:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.nosshkeys_vertica_image.outputs.value }}
+    steps:
+
+    - name: set lower case owner name
+      env:
+        OWNER: '${{ github.repository_owner }}'
+      run: |
+        echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+
+    - name: Pick the name of the image
+      uses: spilchen/switch-case-action@v2
+      id: nosshkeys_vertica_image
+      with:
+        default: ghcr.io/${{ env.OWNER_LC }}/vertica-k8s:${{ github.sha }}-nosshkeys
+        conditionals-with-values: |
+          ${{ inputs.nosshkeys_vertica_image != '' }} => ${{ inputs.nosshkeys_vertica_image }}
+          ${{ github.event_name == 'pull_request' }} => vertica-k8s:kind-nosshkeys
+
+    - name: Login to GitHub Container registry for non-PRs
+      uses: docker/login-action@v2
+      if: ${{ github.event_name != 'pull_request' && inputs.nosshkeys_vertica_image == '' || startsWith(inputs.nosshkeys_vertica_image, 'ghcr.io') }}
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      if: ${{ inputs.nosshkeys_vertica_image != '' && startsWith(inputs.nosshkeys_vertica_image, 'docker.io') }}
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - uses: actions/checkout@v3
+      if: ${{ inputs.nosshkeys_vertica_image == '' }}
+
+    - name: Download the RPM
+      uses: ./.github/actions/download-rpm
+      if: ${{ inputs.nosshkeys_vertica_image == '' }}
+
+    - name: Build and optionally push nosshkeys server image
+      if: ${{ inputs.nosshkeys_vertica_image == '' }}
+      run: |
+        export VERTICA_IMG=${{ steps.nosshkeys_vertica_image.outputs.value }}
+        export NO_SSH_KEYS=yes
+        make docker-build-vertica
+        if [ $GITHUB_EVENT_NAME != 'pull_request' ]
+        then
+          make docker-push-vertica
+        fi
+
+    - name: Save the image for consumption by dependent jobs (PRs only)
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        docker save ${{ steps.nosshkeys_vertica_image.outputs.value }} > nosshkeys-vertica-image.tar
+
+    - uses: actions/upload-artifact@v3
+      if: ${{ github.event_name == 'pull_request' }}
+      with:
+        name: nosshkeys-vertica-image
+        path: nosshkeys-vertica-image.tar
+
+    - name: Do a local pull of the image if we didn't create it
+      if: ${{ inputs.nosshkeys_vertica_image != '' }}
+      run: |
+        docker pull ${{ inputs.nosshkeys_vertica_image }}
+
+    - name: Print a summary of the job
+      run: |
+        echo "Image Name: **${{ steps.nosshkeys_vertica_image.outputs.value }}**" >> $GITHUB_STEP_SUMMARY
+        echo "Was Built: ${{ inputs.nosshkeys_vertica_image == '' && '**Yes**' || '**No**' }}" >> $GITHUB_STEP_SUMMARY
+        echo "Was Pushed: ${{ inputs.nosshkeys_vertica_image == '' && github.event_name != 'pull_request' && '**Yes**' || '**No**' }}"
+        echo "Was Scanned: **No**" >> $GITHUB_STEP_SUMMARY
+        echo "Size: **$(docker inspect --format '{{.Size}}' ${{ steps.nosshkeys_vertica_image.outputs.value }} | numfmt --to=iec)**" >> $GITHUB_STEP_SUMMARY
+        echo "Image ID: **$(docker inspect --format '{{.ID}}' ${{ steps.nosshkeys_vertica_image.outputs.value }})**" >> $GITHUB_STEP_SUMMARY
+        echo "Digest: **$(IFS=":" read image tag <<< $(echo ${{ steps.nosshkeys_vertica_image.outputs.value }} | sed -e 's/^docker.io\///'); docker inspect --format='{{.RepoDigests}}' $image:$tag | sed 's:^.\(.*\).$:\1:' | tr " " "\n" | grep $image | cut -d'@' -f2 || echo "<none>")**" >> $GITHUB_STEP_SUMMARY
+        echo -n "Vertica Version: **" >> $GITHUB_STEP_SUMMARY
+        echo -n $(docker inspect --format '{{index .Config.Labels "vertica-version"}}' ${{ steps.nosshkeys_vertica_image.outputs.value }}) >> $GITHUB_STEP_SUMMARY
+        echo "**" >> $GITHUB_STEP_SUMMARY
 
   build-server-minimal:
     runs-on: ubuntu-latest
@@ -210,7 +300,7 @@ jobs:
 
     - name: Login to Docker Hub
       uses: docker/login-action@v2
-      if: ${{ inputs.full_vertica_image != '' && startsWith(inputs.full_vertica_image, 'docker.io') }}
+      if: ${{ inputs.minimal_vertica_image != '' && startsWith(inputs.minimal_vertica_image, 'docker.io') }}
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/e2e.yml

@@ -14,6 +14,10 @@ on:
         description: 'Name of an existing full vertica image. Leave blank to build one with the default name'
         type: string
         required: false
+      nosshkeys_vertica_image:
+        description: 'Name of an existing vertica image with no ssh keys inside. Leave blank to build one with the default name'
+        type: string
+        required: false
       minimal_vertica_image:
         description: 'Name of an existing minimal vertica image. Leave blank to build one with the default name'
         type: string
@@ -75,6 +79,7 @@ jobs:
       operator_image: ${{ inputs.operator_image }}
       minimal_vertica_image: ${{ inputs.minimal_vertica_image }}
       full_vertica_image: ${{ inputs.full_vertica_image }}
+      nosshkeys_vertica_image: ${{ inputs.nosshkeys_vertica_image }}
       security_scan_exit_code: ${{ inputs.security_scan_exit_code }}
       run_security_scan: ${{ inputs.run_security_scan }}
     secrets:
@@ -125,7 +130,7 @@ jobs:
     with:
       vlogger-image: ${{ needs.build.outputs.vlogger-image }}
       operator-image: ${{ needs.build.outputs.operator-image }}
-      vertica-image: ${{ needs.build.outputs.full-vertica-image }}
+      vertica-image: ${{ needs.build.outputs.nosshkeys-vertica-image }}
     secrets:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

Makefile

@@ -134,6 +134,8 @@ export OLM_CATALOG_IMG
 
 # Set this to YES if you want to create a vertica image of minimal size
 MINIMAL_VERTICA_IMG ?=
+# Set this to YES if you want to create a vertica image with no ssh keys inside
+NO_SSH_KEYS ?=
 # Name of the helm release that we will install/uninstall
 HELM_RELEASE_NAME?=vdb-op
 # Can be used to specify additional overrides when doing the helm install.
@@ -343,7 +345,7 @@ endif
 .PHONY: docker-build-vertica
 docker-build-vertica: docker-vertica/Dockerfile ## Build vertica server docker image
 	cd docker-vertica \
-	&& make VERTICA_IMG=${VERTICA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG}
+	&& make VERTICA_IMG=${VERTICA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG} NO_SSH_KEYS=${NO_SSH_KEYS}
 
 .PHONY: docker-push
 docker-push-vertica:  ## Push vertica server docker image

changes/unreleased/Changed-20230302-084151.yaml

@@ -0,0 +1,6 @@
+kind: Changed
+body: Remove keys from the vertica-k8s container. This will be available in the first
+  server version after 12.0.4.
+time: 2023-03-02T08:41:51.161372267-04:00
+custom:
+  Issue: "343"

docker-vertica/Dockerfile

@@ -6,11 +6,13 @@
 ARG BASE_OS_VERSION="focal-20220801"
 ARG BUILDER_OS_VERSION="7.9.2009"
 ARG MINIMAL=""
+ARG NO_SSH_KEYS=""
 ARG S6_OVERLAY_VERSION=3.1.2.1
 FROM centos:centos${BUILDER_OS_VERSION} as builder
 
 ARG VERTICA_RPM="vertica-x86_64.RHEL6.latest.rpm"
 ARG MINIMAL
+ARG NO_SSH_KEYS
 ARG DBADMIN_GID=5000
 ARG DBADMIN_UID=5000
 
@@ -65,18 +67,26 @@ RUN set -x \
 # Copy in a stable ssh key.  This is done so that Vertica pods can communicate
 # with pods running an older image.  This is necessary when doing an online
 # image change as the Vertica cluster will be running with two container
-# versions at once.
+# versions at once. This step is required if not including SSH keys because
+# conditional copy isn't a thing in Docker. If no SSH keys are to be included, 
+# then the next RUN will remove the key we just added.
 COPY dbadmin/.ssh /home/dbadmin/.ssh
 
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN set -x \
   && mkdir -p /root/.ssh \
-  && cp -r /home/dbadmin/.ssh /root \
-  && chmod 700 /root/.ssh \
-  && chmod 600 /root/.ssh/* \
+  && if [[ ${NO_SSH_KEYS^^} != "YES" ]] ; then \
+    cp -r /home/dbadmin/.ssh /root; \
+    chmod 700 /root/.ssh; \
+    chmod 600 /root/.ssh/*; \
+  fi \
   && chmod 700 /home/dbadmin/.ssh \
   && chmod 600 /home/dbadmin/.ssh/* \
   && chown -R dbadmin:verticadba /home/dbadmin/ \
-  && chmod go-w /etc/ssh/sshd_config.d/* /etc/ssh/ssh_config.d/*
+  && chmod go-w /etc/ssh/sshd_config.d/* /etc/ssh/ssh_config.d/* \
+  && if [[ ${NO_SSH_KEYS^^} == "YES" ]] ; then \
+    rm -rf /home/dbadmin/.ssh/*;  \
+  fi
 
 ##############################################################################################
 FROM ubuntu:${BASE_OS_VERSION}
@@ -138,29 +148,30 @@ RUN set -x \
   sudo \
   vim-tiny \
   # Install jre if not minimal
-  && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") ]] ; then \
+  && if [[ ${MINIMAL^^} != "YES" ]] ; then \
     apt-get install -y --no-install-recommends $JRE_PKG; \
   fi \
   && rm -rf /var/lib/apt/lists/* \
   # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default
   && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
   && mkdir -p /run/sshd \
-  && ssh-keygen -q -A \
   && /usr/sbin/groupadd -r verticadba --gid ${DBADMIN_GID} \
   && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \
   # Allow passwordless sudo access from dbadmin
   && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \
   && echo "dbadmin -       nofile  65536" >> /etc/security/limits.conf \
   # Set JAVA_HOME environment variable if not minimal, this will be loaded to all shells
-  && if [[ $MINIMAL != "YES" && $MINIMAL != "yes" ]]; then \
+  && if [[ ${MINIMAL^^} != "YES" ]] ; then \
     echo "JAVA_HOME=/usr" >> /etc/environment; \
   fi \
   # Create a symlink to python3 interpreter in vertica
   && update-alternatives --install /usr/bin/python python /opt/vertica/oss/python3/bin/python3 1 \
   && chmod u+s /usr/sbin/cron \
   # Untar the init program that was downloaded earlier
   && tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz \
-  && tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz
+  && tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz \
+  # delete old host keys
+  && rm -rf /etc/ssh/ssh_host* 
 
 ENTRYPOINT [ "/init" ]
 

docker-vertica/Makefile

@@ -3,6 +3,7 @@ BUILDER_OS_VERSION?=7.9.2009
 BASE_OS_VERSION?=focal-20220316
 VERTICA_IMG?=vertica-k8s
 MINIMAL_VERTICA_IMG?=
+NO_SSH_KEYS?=
 VERTICA_VERSION?=$(shell rpm --nosignature -qp --queryformat '%{VERSION}-%{RELEASE}' packages/$(VERTICA_RPM))
 
 all: docker-build-vertica
@@ -16,6 +17,7 @@ docker-build-vertica: Dockerfile packages/package-checksum-patcher.py
 		--label vertica-version=${VERTICA_VERSION} \
 		--build-arg MINIMAL=${MINIMAL_VERTICA_IMG} \
 		--build-arg VERTICA_RPM=${VERTICA_RPM} \
+		--build-arg NO_SSH_KEYS=${NO_SSH_KEYS} \
 		--build-arg BASE_OS_VERSION=${BASE_OS_VERSION} \
 		--build-arg BUILDER_OS_VERSION=${BUILDER_OS_VERSION} \
 		-t ${VERTICA_IMG} .

docker-vertica/packages/cleanup.sh

@@ -23,6 +23,8 @@
 #
 # wander around in the image looking for things you can remove
 rm -r -f \
+   /opt/vertica/config/https_cert/*.key \
+   /opt/vertica/config/share/agent* \
    /opt/vertica/examples \
    /opt/vertica/packages/*/examples \
    /opt/vertica/oss/python*/lib/python*/test \
@@ -33,6 +35,9 @@ rm -r -f \
    /opt/vertica/oss/python*/lib/python*/tkinter \
    /opt/vertica/oss/python*/lib/python*/idlelib
 
+# cleanup all test directories for packages under site-package
+find /opt/vertica/oss/python*/lib/python*/site-packages/ -type d -name "*[Tt]est" -exec rm -rf {} +
+
 # cleanup many of the __pycache__ directories 
 find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} +
 

docker-vertica/s6-rc.d/sshd/run

@@ -1,2 +1,4 @@
 #!/command/execlineb -P
-sudo /usr/sbin/sshd
+foreground { sudo ssh-keygen -q -A } sudo /usr/sbin/sshd
+
+

tests/e2e-leg-1/auto-restart-vertica/09-create-ssh-keys.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: "mkdir -p ssh-keys && rm -f ssh-keys/* && ssh-keygen -q -t rsa -N '' -f ssh-keys/id_rsa && cp ssh-keys/id_rsa.pub ssh-keys/authorized_keys"
+  - script: "kubectl delete secret -n $NAMESPACE ssh-keys || :"
+  - script: "kubectl create secret -n $NAMESPACE generic ssh-keys --from-file=ssh-keys"

tests/e2e-leg-1/auto-restart-vertica/setup-vdb/base/setup-vdb.yaml

@@ -18,6 +18,7 @@ metadata:
 spec:
   initPolicy: CreateSkipPackageInstall
   image: kustomize-vertica-image
+  sshSecret: ssh-keys
   sidecars:
     - name: vlogger
       image: kustomize-vlogger-image

tests/e2e-leg-1/autoscale-by-pod/14-create-ssh-keys.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: "mkdir -p ssh-keys && rm -f ssh-keys/* && ssh-keygen -q -t rsa -N '' -f ssh-keys/id_rsa && cp ssh-keys/id_rsa.pub ssh-keys/authorized_keys"
+  - script: "kubectl delete secret -n $NAMESPACE ssh-keys || :"
+  - script: "kubectl create secret -n $NAMESPACE generic ssh-keys --from-file=ssh-keys"

tests/e2e-leg-1/autoscale-by-pod/setup-vdb/base/setup-vdb.yaml

@@ -18,6 +18,7 @@ metadata:
 spec:
   initPolicy: CreateSkipPackageInstall
   image: kustomize-vertica-image
+  sshSecret: ssh-keys
   communal:
     includeUIDInPath: true
   local:

tests/e2e-leg-1/autoscale-by-subcluster/14-create-ssh-keys.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: "mkdir -p ssh-keys && rm -f ssh-keys/* && ssh-keygen -q -t rsa -N '' -f ssh-keys/id_rsa && cp ssh-keys/id_rsa.pub ssh-keys/authorized_keys"
+  - script: "kubectl delete secret -n $NAMESPACE ssh-keys || :"
+  - script: "kubectl create secret -n $NAMESPACE generic ssh-keys --from-file=ssh-keys"

tests/e2e-leg-1/autoscale-by-subcluster/setup-vdb/base/setup-vdb.yaml

@@ -18,6 +18,7 @@ metadata:
 spec:
   initPolicy: CreateSkipPackageInstall
   image: kustomize-vertica-image
+  sshSecret: ssh-keys
   communal:
     includeUIDInPath: true
   local: