Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make TestKanikoTaskRun more portable #1516

Merged
merged 2 commits into from
Nov 8, 2019
Merged

Make TestKanikoTaskRun more portable #1516

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d06b97
Make TestKanikoTaskRun more portable
vdemeester Nov 4, 2019
5dfe457
Refactor setup to take extra setup functions ⛽
vdemeester Nov 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions test/build_logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ import (

// CollectPodLogs will get the logs for all containers in a Pod
func CollectPodLogs(c *clients, podName, namespace string, logf logging.FormatLogger) {
logs, err := getContainerLogsFromPod(c.KubeClient.Kube, podName, namespace)
logs, err := getContainersLogsFromPod(c.KubeClient.Kube, podName, namespace)
if err != nil {
logf("Could not get logs for pod %s: %s", podName, err)
}
logf("build logs %s", logs)
}

func getContainerLogsFromPod(c kubernetes.Interface, pod, namespace string) (string, error) {
func getContainersLogsFromPod(c kubernetes.Interface, pod, namespace string) (string, error) {
p, err := c.CoreV1().Pods(namespace).Get(pod, metav1.GetOptions{})
if err != nil {
return "", err
Expand All @@ -45,16 +45,26 @@ func getContainerLogsFromPod(c kubernetes.Interface, pod, namespace string) (str
sb := strings.Builder{}
for _, container := range p.Spec.Containers {
sb.WriteString(fmt.Sprintf("\n>>> Container %s:\n", container.Name))
req := c.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{Follow: true, Container: container.Name})
rc, err := req.Stream()
logs, err := getContainerLogsFromPod(c, pod, container.Name, namespace)
if err != nil {
return "", err
}
bs, err := ioutil.ReadAll(rc)
if err != nil {
return "", err
}
sb.Write(bs)
sb.WriteString(logs)
}
return sb.String(), nil
}

func getContainerLogsFromPod(c kubernetes.Interface, pod, container, namespace string) (string, error) {
sb := strings.Builder{}
req := c.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{Follow: true, Container: container})
rc, err := req.Stream()
if err != nil {
return "", err
}
bs, err := ioutil.ReadAll(rc)
if err != nil {
return "", err
}
sb.Write(bs)
return sb.String(), nil
}
144 changes: 109 additions & 35 deletions test/kaniko_task_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,17 @@ package test

import (
"fmt"
"strings"
"testing"
"time"

"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-cmp/cmp"
"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
tb "github.com/tektoncd/pipeline/test/builder"
"golang.org/x/xerrors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
knativetest "knative.dev/pkg/test"
)

Expand All @@ -45,16 +45,34 @@ const (

// TestTaskRun is an integration test that will verify a TaskRun using kaniko
func TestKanikoTaskRun(t *testing.T) {
repo := ensureDockerRepo(t)
c, namespace := setup(t)
t.Parallel()

repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace)

knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
defer tearDown(t, c, namespace)

hasSecretConfig, err := CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, "kaniko-secret")
if err != nil {
t.Fatalf("Expected to create kaniko creds: %v", err)
if _, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Create(getRegistryDeployment(namespace)); err != nil {
t.Fatalf("Failed to create the local registry deployment: %v", err)
}
service := getRegistryService(namespace)
if _, err := c.KubeClient.Kube.CoreV1().Services(namespace).Create(service); err != nil {
t.Fatalf("Failed to create the local registry service: %v", err)
}
set := labels.Set(service.Spec.Selector)
if pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil {
t.Fatalf("Failed to list Pods of service[%s] error:%v", service.GetName(), err)
} else {
if len(pods.Items) != 1 {
t.Fatalf("Only 1 pod for service %s should be running: %v", service, pods.Items)
}

if err := WaitForPodState(c, pods.Items[0].Name, namespace, func(pod *corev1.Pod) (bool, error) {
return pod.Status.Phase == "Running", nil
}, "PodContainersRunning"); err != nil {
t.Fatalf("Error waiting for Pod %q to run: %v", pods.Items[0].Name, err)
}
}

t.Logf("Creating Git PipelineResource %s", kanikoGitResourceName)
Expand All @@ -68,7 +86,7 @@ func TestKanikoTaskRun(t *testing.T) {
}

t.Logf("Creating Task %s", kanikoTaskName)
if _, err := c.TaskClient.Create(getTask(repo, namespace, hasSecretConfig)); err != nil {
if _, err := c.TaskClient.Create(getTask(repo, namespace)); err != nil {
t.Fatalf("Failed to create Task `%s`: %s", kanikoTaskName, err)
}

Expand Down Expand Up @@ -111,12 +129,58 @@ func TestKanikoTaskRun(t *testing.T) {
}

// match the local digest, which is first capture group against the remote image
remoteDigest, err := getRemoteDigest(repo)
remoteDigest, err := getRemoteDigest(t, c, namespace, repo)
if err != nil {
t.Fatalf("Expected to get digest for remote image %s", repo)
t.Fatalf("Expected to get digest for remote image %s: %v", repo, err)
}
if d := cmp.Diff(digest, remoteDigest); d != "" {
t.Fatalf("Expected local digest %s to match remote digest %s: %s", digest, remoteDigest, d)
}
}

func getRegistryDeployment(namespace string) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: "registry",
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "registry",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "registry",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Name: "registry",
Image: "registry",
}},
},
},
},
}
if digest != remoteDigest {
t.Fatalf("Expected local digest %s to match remote digest %s", digest, remoteDigest)
}

func getRegistryService(namespace string) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: "registry",
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{{
Port: 5000,
}},
Selector: map[string]string{
"app": "registry",
},
},
}
}

Expand All @@ -135,7 +199,7 @@ func getImageResource(namespace, repo string) *v1alpha1.PipelineResource {
))
}

func getTask(repo, namespace string, withSecretConfig bool) *v1alpha1.Task {
func getTask(repo, namespace string) *v1alpha1.Task {
taskSpecOps := []tb.TaskSpecOp{
tb.TaskInputs(tb.InputsResource("gitsource", v1alpha1.PipelineResourceTypeGit)),
tb.TaskOutputs(tb.OutputsResource("builtImage", v1alpha1.PipelineResourceTypeImage)),
Expand All @@ -146,21 +210,15 @@ func getTask(repo, namespace string, withSecretConfig bool) *v1alpha1.Task {
fmt.Sprintf("--destination=%s", repo),
"--context=/workspace/gitsource",
"--oci-layout-path=/workspace/output/builtImage",
"--insecure",
"--insecure-pull",
"--insecure-registry=registry"+namespace+":5000/",
),
}
if withSecretConfig {
stepOps = append(stepOps,
tb.StepVolumeMount("kaniko-secret", "/secrets"),
tb.StepEnvVar("GOOGLE_APPLICATION_CREDENTIALS", "/secrets/config.json"),
)
taskSpecOps = append(taskSpecOps, tb.TaskVolume("kaniko-secret", tb.VolumeSource(corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "kaniko-secret",
},
})))
}
step := tb.Step("kaniko", "gcr.io/kaniko-project/executor:v0.13.0", stepOps...)
taskSpecOps = append(taskSpecOps, step)
sidecar := tb.Sidecar("registry", "registry")
taskSpecOps = append(taskSpecOps, sidecar)

return tb.Task(kanikoTaskName, namespace, tb.TaskSpec(taskSpecOps...))
}
Expand All @@ -174,18 +232,34 @@ func getTaskRun(namespace string) *v1alpha1.TaskRun {
))
}

func getRemoteDigest(image string) (string, error) {
ref, err := name.ParseReference(image, name.WeakValidation)
if err != nil {
return "", xerrors.Errorf("could not parse image reference %q: %w", image, err)
func getRemoteDigest(t *testing.T, c *clients, namespace, image string) (string, error) {
t.Helper()
podName := "skopeo-jq"
afrittoli marked this conversation as resolved.
Show resolved Hide resolved
if _, err := c.KubeClient.Kube.CoreV1().Pods(namespace).Create(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: podName,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Name: "skopeo",
Image: "gcr.io/tekton-releases/dogfooding/skopeo:latest",
Command: []string{"/bin/sh", "-c"},
Args: []string{"skopeo inspect --tls-verify=false docker://" + image + ":latest| jq '.Digest'"},
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i feel like this and/or the function could use some comments to explain what skopeo is, what the function is all about and how exactly it determines the digest (looks like specifically looking for image with tag latest - which would mean if you don't tag latest, it wont work)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed, I'll follow-up on this to add the comment on the function 👼
Also I feel the function name could be more explicit (like getRemoteDigestFromTheInside or something like that 😹)

}},
RestartPolicy: corev1.RestartPolicyNever,
},
}); err != nil {
t.Fatalf("Failed to create the local registry service: %v", err)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The log message doesn't look right here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh yeah, my bad =)

}
img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
return "", xerrors.Errorf("could not pull remote ref %s: %w", ref, err)
if err := WaitForPodState(c, podName, namespace, func(pod *corev1.Pod) (bool, error) {
return pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed", nil
}, "PodContainersTerminated"); err != nil {
t.Fatalf("Error waiting for Pod %q to terminate: %v", podName, err)
}
digest, err := img.Digest()
logs, err := getContainerLogsFromPod(c.KubeClient.Kube, podName, "skopeo", namespace)
if err != nil {
return "", xerrors.Errorf("could not get digest for image %s: %w", img, err)
t.Fatalf("Could not get logs for pod %s: %s", podName, err)
}
return digest.String(), nil
return strings.TrimSpace(strings.ReplaceAll(logs, "\"", "")), nil
}