Skip to content

Commit

Permalink
Merge pull request #1774 from tnozicka/collect-ephemeral
Browse files Browse the repository at this point in the history 
Collect logs from ephemeral containers in CI and must-gather
  • Loading branch information
@scylla-operator-bot
scylla-operator-bot[bot] authored Feb 29, 2024
2 parents 8933502 + 0db1c0e commit 0308d14
Show file tree
Hide file tree
Showing 2 changed files with 166 additions and 11 deletions.
29 changes: 18 additions & 11 deletions pkg/gather/collect/collect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,14 +237,14 @@ func retrieveContainerLogs(ctx context.Context, podClient corev1client.PodInterf
return nil
}

func (c *Collector) collectContainerLogs(ctx context.Context, logsDir string, pod *corev1.Pod, container *corev1.Container) error {
func (c *Collector) collectContainerLogs(ctx context.Context, logsDir string, podMeta *metav1.ObjectMeta, podCSs []corev1.ContainerStatus, containerName string) error {
var err error

cs, _, found := slices.Find(pod.Status.ContainerStatuses, func(s corev1.ContainerStatus) bool {
return s.Name == container.Name
cs, _, found := slices.Find(podCSs, func(s corev1.ContainerStatus) bool {
return s.Name == containerName
})
if !found {
klog.InfoS("Container doesn't yet have a status", "Pod", naming.ObjRef(pod), "Container", container.Name)
klog.InfoS("Container doesn't yet have a status", "Pod", naming.ObjRef(podMeta), "Container", containerName)
return nil
}

Expand All @@ -254,7 +254,7 @@ func (c *Collector) collectContainerLogs(ctx context.Context, logsDir string, po
}

logOptions := &corev1.PodLogOptions{
Container: container.Name,
Container: containerName,
Timestamps: true,
Follow: false,
LimitBytes: limitBytes,
Expand All @@ -267,17 +267,17 @@ func (c *Collector) collectContainerLogs(ctx context.Context, logsDir string, po
if cs.State.Running != nil {
// Retrieve current logs.
logOptions.Previous = false
err = retrieveContainerLogs(ctx, c.corev1Client.Pods(pod.Namespace), filepath.Join(logsDir, container.Name+".current"), pod.Name, logOptions)
err = retrieveContainerLogs(ctx, c.corev1Client.Pods(podMeta.Namespace), filepath.Join(logsDir, containerName+".current"), podMeta.Name, logOptions)
if err != nil {
return fmt.Errorf("can't retrieve pod logs for container %q in pod %q: %w", container.Name, naming.ObjRef(pod), err)
return fmt.Errorf("can't retrieve pod logs for container %q in pod %q: %w", containerName, naming.ObjRef(podMeta), err)
}
}

if cs.LastTerminationState.Terminated != nil {
logOptions.Previous = true
err = retrieveContainerLogs(ctx, c.corev1Client.Pods(pod.Namespace), filepath.Join(logsDir, container.Name+".previous"), pod.Name, logOptions)
err = retrieveContainerLogs(ctx, c.corev1Client.Pods(podMeta.Namespace), filepath.Join(logsDir, containerName+".previous"), podMeta.Name, logOptions)
if err != nil {
return fmt.Errorf("can't retrieve previous pod logs for container %q in pod %q: %w", container.Name, naming.ObjRef(pod), err)
return fmt.Errorf("can't retrieve previous pod logs for container %q in pod %q: %w", containerName, naming.ObjRef(podMeta), err)
}
}

Expand Down Expand Up @@ -308,19 +308,26 @@ func (c *Collector) collectPod(ctx context.Context, u *unstructured.Unstructured
}

for _, container := range pod.Spec.InitContainers {
err = c.collectContainerLogs(ctx, logsDir, pod, &container)
err = c.collectContainerLogs(ctx, logsDir, &pod.ObjectMeta, pod.Status.InitContainerStatuses, container.Name)
if err != nil {
return fmt.Errorf("can't collect logs for init container %q in pod %q: %w", container.Name, naming.ObjRef(pod), err)
}
}

for _, container := range pod.Spec.Containers {
err = c.collectContainerLogs(ctx, logsDir, pod, &container)
err = c.collectContainerLogs(ctx, logsDir, &pod.ObjectMeta, pod.Status.ContainerStatuses, container.Name)
if err != nil {
return fmt.Errorf("can't collect logs for container %q in pod %q: %w", container.Name, naming.ObjRef(pod), err)
}
}

for _, container := range pod.Spec.EphemeralContainers {
err = c.collectContainerLogs(ctx, logsDir, &pod.ObjectMeta, pod.Status.EphemeralContainerStatuses, container.Name)
if err != nil {
return fmt.Errorf("can't collect logs for ephemeral container %q in pod %q: %w", container.Name, naming.ObjRef(pod), err)
}
}

return nil
}

Expand Down
148 changes: 148 additions & 0 deletions pkg/gather/collect/collect_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,34 @@ status:
Name: "my-pod",
},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{
{
Name: "my-init-container",
},
},
Containers: []corev1.Container{
{
Name: "my-container",
},
},
EphemeralContainers: []corev1.EphemeralContainer{
{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "my-ephemeral-container",
},
},
},
},
Status: corev1.PodStatus{
InitContainerStatuses: []corev1.ContainerStatus{
{
Name: "my-init-container",
State: corev1.ContainerState{
Terminated: nil,
Running: &corev1.ContainerStateRunning{},
},
},
},
ContainerStatuses: []corev1.ContainerStatus{
{
Name: "my-container",
Expand All @@ -176,6 +197,15 @@ status:
},
},
},
EphemeralContainerStatuses: []corev1.ContainerStatus{
{
Name: "my-ephemeral-container",
State: corev1.ContainerState{
Terminated: nil,
Running: &corev1.ContainerStateRunning{},
},
},
},
},
},
existingObjects: nil,
Expand All @@ -198,6 +228,12 @@ spec:
containers:
- name: my-container
resources: {}
ephemeralContainers:
- name: my-ephemeral-container
resources: {}
initContainers:
- name: my-init-container
resources: {}
status:
containerStatuses:
- image: ""
Expand All @@ -209,12 +245,40 @@ status:
state:
running:
startedAt: null
ephemeralContainerStatuses:
- image: ""
imageID: ""
lastState: {}
name: my-ephemeral-container
ready: false
restartCount: 0
state:
running:
startedAt: null
initContainerStatuses:
- image: ""
imageID: ""
lastState: {}
name: my-init-container
ready: false
restartCount: 0
state:
running:
startedAt: null
`, "\n"),
},
{
Name: "namespaces/test/pods/my-pod/my-container.current",
Content: "fake logs",
},
{
Name: "namespaces/test/pods/my-pod/my-ephemeral-container.current",
Content: "fake logs",
},
{
Name: "namespaces/test/pods/my-pod/my-init-container.current",
Content: "fake logs",
},
},
},
},
Expand All @@ -226,11 +290,23 @@ status:
Name: "my-pod",
},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{
{
Name: "my-init-container",
},
},
Containers: []corev1.Container{
{
Name: "my-container",
},
},
EphemeralContainers: []corev1.EphemeralContainer{
{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "my-ephemeral-container",
},
},
},
},
Status: corev1.PodStatus{
ContainerStatuses: []corev1.ContainerStatus{
Expand All @@ -244,6 +320,28 @@ status:
},
},
},
InitContainerStatuses: []corev1.ContainerStatus{
{
Name: "my-init-container",
State: corev1.ContainerState{
Running: &corev1.ContainerStateRunning{},
},
LastTerminationState: corev1.ContainerState{
Terminated: &corev1.ContainerStateTerminated{},
},
},
},
EphemeralContainerStatuses: []corev1.ContainerStatus{
{
Name: "my-ephemeral-container",
State: corev1.ContainerState{
Running: &corev1.ContainerStateRunning{},
},
LastTerminationState: corev1.ContainerState{
Terminated: &corev1.ContainerStateTerminated{},
},
},
},
},
},
existingObjects: nil,
Expand All @@ -266,6 +364,12 @@ spec:
containers:
- name: my-container
resources: {}
ephemeralContainers:
- name: my-ephemeral-container
resources: {}
initContainers:
- name: my-init-container
resources: {}
status:
containerStatuses:
- image: ""
Expand All @@ -281,6 +385,34 @@ status:
state:
running:
startedAt: null
ephemeralContainerStatuses:
- image: ""
imageID: ""
lastState:
terminated:
exitCode: 0
finishedAt: null
startedAt: null
name: my-ephemeral-container
ready: false
restartCount: 0
state:
running:
startedAt: null
initContainerStatuses:
- image: ""
imageID: ""
lastState:
terminated:
exitCode: 0
finishedAt: null
startedAt: null
name: my-init-container
ready: false
restartCount: 0
state:
running:
startedAt: null
`, "\n"),
},
{
Expand All @@ -291,6 +423,22 @@ status:
Name: "namespaces/test/pods/my-pod/my-container.previous",
Content: "fake logs",
},
{
Name: "namespaces/test/pods/my-pod/my-ephemeral-container.current",
Content: "fake logs",
},
{
Name: "namespaces/test/pods/my-pod/my-ephemeral-container.previous",
Content: "fake logs",
},
{
Name: "namespaces/test/pods/my-pod/my-init-container.current",
Content: "fake logs",
},
{
Name: "namespaces/test/pods/my-pod/my-init-container.previous",
Content: "fake logs",
},
},
},
},
Expand Down

0 comments on commit 0308d14

Please sign in to comment.