Changes to allow core/vstack in the container (#230)

There are two changes in the PR to aid in debugging:

- Added the SYS_PTRACE capability and gdb to the container. This allows us to
  run vstack to collect stacks of the vertica process.
- Added an option to allow a custom securityContext to be specified. This was
  necessary since you need to run in privileged mode to set the core ulimit in
  order to collect core files in the container. Note, by default the vertica
  container still does not run in privileged mode. This new option is just to
  allow us to run in that mode.

To capture core files, the following needs to be done:

1. Set the privileged mode in the container. This can be set now with the
securityContext parameter like this:

apiVersion: vertica.com/v1beta1
kind: VerticaDB
metadata:
  name: v
spec:
  image: "vertica/vertica-k8s:12.0.0-0-minimal"
  communal:
    path: "s3://nimbusdb/db"
    endpoint: "http://minio"
    credentialSecret: s3-auth
    includeUIDInPath: true
  kSafety: "0"
  securityContext:
    privileged: true
  subclusters:
    - name: defaultsubcluster
      size: 1

2. Ensure /proc/sys/kernel/core_pattern is set to core in the host machine.
This file is not namespaced, so when setting this value in the host it effects
all containers running on that host.

$ cat /proc/sys/kernel/core_pattern
core

If vertica does generate a core, a message is written to the vertica.log to
indicate where the file is written too.

api/v1beta1/verticadb_types.go

@@ -302,6 +302,14 @@ type VerticaDBSpec struct {
 	// creation of the CR.  If set for initPolicy other than Create, then it has
 	// no effect.
 	EncryptSpreadComm string `json:"encryptSpreadComm,omitempty"`
+
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:hidden"
+	// Allows you to set any additional securityContext for the Vertica server
+	// container.  We merge the values with the securityContext generated by the
+	// operator.  The operator adds its own capabilities to this.  If you want
+	// those capabilities to be removed you must explicitly include them in the
+	// drop list.
+	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
 }
 
 // LocalObjectReference is used instead of corev1.LocalObjectReference and behaves the same.

changes/unreleased/Added-20220704-163049.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: Allow vstack and cores to be taken in the container
+time: 2022-07-04T16:30:49.880373536-03:00
+custom:
+  Issue: "230"

docker-vertica/Dockerfile

@@ -111,21 +111,22 @@ RUN set -x \
   # fixed libraries. 
   && apt-get -y update \
   && apt-get install -y --no-install-recommends \
-  locales \
-  cron \
-  sysstat \
   ca-certificates \
-  ntp \
+  cron \
   dialog \
+  gdb \
   iproute2 \
+  krb5-user \
   libkeyutils1\
   libz-dev \
-  procps \
-  krb5-user \
+  locales \
   logrotate \
+  ntp \
   openssh-client \
   openssh-server \
   openssl \
+  procps \
+  sysstat \
   sudo \
   && rm -rf /var/lib/apt/lists/* \
   # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default

pkg/builder/builder.go

@@ -414,15 +414,48 @@ func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Contai
 				},
 			},
 		},
-		// Is needed to run sshd on Openshift
-		SecurityContext: &corev1.SecurityContext{
-			Capabilities: &corev1.Capabilities{
-				Add: []corev1.Capability{"SYS_CHROOT", "AUDIT_WRITE"},
-			},
-		},
-		Env:          envVars,
-		VolumeMounts: buildVolumeMounts(vdb),
+		SecurityContext: makeServerSecurityContext(vdb),
+		Env:             envVars,
+		VolumeMounts:    buildVolumeMounts(vdb),
+	}
+}
+
+func makeServerSecurityContext(vdb *vapi.VerticaDB) *corev1.SecurityContext {
+	sc := &corev1.SecurityContext{}
+	if vdb.Spec.SecurityContext != nil {
+		sc = vdb.Spec.SecurityContext
+	}
+	if sc.Capabilities == nil {
+		sc.Capabilities = &corev1.Capabilities{}
+	}
+	capabilitiesNeeded := []corev1.Capability{
+		// Needed to run sshd on OpenShift
+		"SYS_CHROOT",
+		// Needed to run sshd on OpenShift
+		"AUDIT_WRITE",
+		// Needed to be able to collect stacks via vstack
+		"SYS_PTRACE",
+	}
+	for i := range capabilitiesNeeded {
+		foundCap := false
+		for j := range sc.Capabilities.Add {
+			if capabilitiesNeeded[i] == sc.Capabilities.Add[j] {
+				foundCap = true
+				break
+			}
+		}
+		for j := range sc.Capabilities.Drop {
+			if capabilitiesNeeded[i] == sc.Capabilities.Drop[j] {
+				// If the capability we want to add is *dropped*, we won't bother adding it in
+				foundCap = false
+				break
+			}
+		}
+		if !foundCap {
+			sc.Capabilities.Add = append(sc.Capabilities.Add, capabilitiesNeeded[i])
+		}
 	}
+	return sc
 }
 
 // makeContainers creates the list of containers to include in the pod spec.

pkg/builder/builder_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	vapi "github.com/vertica/vertica-kubernetes/api/v1beta1"
+	v1 "k8s.io/api/core/v1"
 )
 
 var _ = Describe("builder", func() {
@@ -40,4 +41,38 @@ var _ = Describe("builder", func() {
 			Expect(reflect.DeepEqual(c, baseContainer)).Should(BeTrue())
 		}
 	})
+
+	It("should add our own capabilities to the securityContext", func() {
+		vdb := vapi.MakeVDB()
+		baseContainer := makeServerContainer(vdb, &vdb.Spec.Subclusters[0])
+		Expect(baseContainer.SecurityContext).ShouldNot(BeNil())
+		Expect(baseContainer.SecurityContext.Capabilities).ShouldNot(BeNil())
+		Expect(baseContainer.SecurityContext.Capabilities.Add).Should(ContainElements([]v1.Capability{"SYS_CHROOT", "AUDIT_WRITE", "SYS_PTRACE"}))
+	})
+
+	It("should add omit our own capabilities in the securityContext if we are dropping them", func() {
+		vdb := vapi.MakeVDB()
+		vdb.Spec.SecurityContext = &v1.SecurityContext{
+			Capabilities: &v1.Capabilities{
+				Drop: []v1.Capability{"AUDIT_WRITE"},
+			},
+		}
+		baseContainer := makeServerContainer(vdb, &vdb.Spec.Subclusters[0])
+		Expect(baseContainer.SecurityContext).ShouldNot(BeNil())
+		Expect(baseContainer.SecurityContext.Capabilities).ShouldNot(BeNil())
+		Expect(baseContainer.SecurityContext.Capabilities.Add).Should(ContainElements([]v1.Capability{"SYS_CHROOT", "SYS_PTRACE"}))
+		Expect(baseContainer.SecurityContext.Capabilities.Add).ShouldNot(ContainElement([]v1.Capability{"AUDIT_WRITE"}))
+	})
+
+	It("should allow you to run in priv mode", func() {
+		vdb := vapi.MakeVDB()
+		priv := true
+		vdb.Spec.SecurityContext = &v1.SecurityContext{
+			Privileged: &priv,
+		}
+		baseContainer := makeServerContainer(vdb, &vdb.Spec.Subclusters[0])
+		Expect(baseContainer.SecurityContext).ShouldNot(BeNil())
+		Expect(baseContainer.SecurityContext.Privileged).ShouldNot(BeNil())
+		Expect(*baseContainer.SecurityContext.Privileged).Should(BeTrue())
+	})
 })

pkg/builder/suite_test.go

@@ -0,0 +1,32 @@
+/*
+ (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package builder
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+)
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	RunSpecsWithDefaultAndCustomReporters(t,
+		"builder Suite",
+		[]Reporter{printer.NewlineReporter{}})
+}

tests/e2e/client-access/45-assert.yaml

@@ -0,0 +1,23 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-verify-vstack
+status:
+  containerStatuses:
+    - name: test
+      state:
+        terminated:
+          exitCode: 0

tests/e2e/client-access/45-verify-vstack.yaml

@@ -0,0 +1,49 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: script-verify-vstack
+data:
+  entrypoint.sh: |-
+    #!/bin/bash
+    set -o xtrace
+    set -o errexit
+
+    VSTACK_OP=$(kubectl exec svc/v-client-access-sc1 -- sudo /opt/vertica/bin/vstack)
+    echo $VSTACK_OP
+    echo $VSTACK_OP | grep -cq 'Thread'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-verify-vstack
+  labels:
+    stern: include
+spec:
+  restartPolicy: Never
+  containers:
+    - name: test
+      image: quay.io/helmpack/chart-testing:v3.3.1
+      command: ["/bin/entrypoint.sh"]
+      volumeMounts:
+        - name: entrypoint-volume
+          mountPath: /bin/entrypoint.sh
+          readOnly: true
+          subPath: entrypoint.sh
+  volumes:
+    - name: entrypoint-volume
+      configMap:
+        defaultMode: 0700
+        name: script-verify-vstack