From 7d06b97aa6aaa8e19433431b11c69ceedafde97b Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 4 Nov 2019 14:59:09 +0100 Subject: [PATCH 1/2] Make TestKanikoTaskRun more portable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test could, previously, only run on specific environment, and would depend on GCP on the CI. This is not more the case with this changes as it is now able to run on any kubernetes. We start a registry in the same namespace the test happens, and we push build/push the image on this local registry. To validate the digest, we need to run a new pod using `skopeo` and `jq` to get the remote digest. Signed-off-by: Vincent Demeester --- test/build_logs.go | 28 +++++--- test/kaniko_task_test.go | 144 +++++++++++++++++++++++++++++---------- 2 files changed, 128 insertions(+), 44 deletions(-) diff --git a/test/build_logs.go b/test/build_logs.go index 65c0cbf744d..9b7eac9b0b0 100644 --- a/test/build_logs.go +++ b/test/build_logs.go @@ -29,14 +29,14 @@ import ( // CollectPodLogs will get the logs for all containers in a Pod func CollectPodLogs(c *clients, podName, namespace string, logf logging.FormatLogger) { - logs, err := getContainerLogsFromPod(c.KubeClient.Kube, podName, namespace) + logs, err := getContainersLogsFromPod(c.KubeClient.Kube, podName, namespace) if err != nil { logf("Could not get logs for pod %s: %s", podName, err) } logf("build logs %s", logs) } -func getContainerLogsFromPod(c kubernetes.Interface, pod, namespace string) (string, error) { +func getContainersLogsFromPod(c kubernetes.Interface, pod, namespace string) (string, error) { p, err := c.CoreV1().Pods(namespace).Get(pod, metav1.GetOptions{}) if err != nil { return "", err @@ -45,16 +45,26 @@ func getContainerLogsFromPod(c kubernetes.Interface, pod, namespace string) (str sb := strings.Builder{} for _, container := range p.Spec.Containers { sb.WriteString(fmt.Sprintf("\n>>> Container %s:\n", container.Name)) - req := c.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{Follow: true, Container: container.Name}) - rc, err := req.Stream() + logs, err := getContainerLogsFromPod(c, pod, container.Name, namespace) if err != nil { return "", err } - bs, err := ioutil.ReadAll(rc) - if err != nil { - return "", err - } - sb.Write(bs) + sb.WriteString(logs) + } + return sb.String(), nil +} + +func getContainerLogsFromPod(c kubernetes.Interface, pod, container, namespace string) (string, error) { + sb := strings.Builder{} + req := c.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{Follow: true, Container: container}) + rc, err := req.Stream() + if err != nil { + return "", err + } + bs, err := ioutil.ReadAll(rc) + if err != nil { + return "", err } + sb.Write(bs) return sb.String(), nil } diff --git a/test/kaniko_task_test.go b/test/kaniko_task_test.go index 59925fa81aa..18399a8e14f 100644 --- a/test/kaniko_task_test.go +++ b/test/kaniko_task_test.go @@ -20,17 +20,17 @@ package test import ( "fmt" + "strings" "testing" "time" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-cmp/cmp" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" tb "github.com/tektoncd/pipeline/test/builder" - "golang.org/x/xerrors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" knativetest "knative.dev/pkg/test" ) @@ -45,16 +45,34 @@ const ( // TestTaskRun is an integration test that will verify a TaskRun using kaniko func TestKanikoTaskRun(t *testing.T) { - repo := ensureDockerRepo(t) c, namespace := setup(t) t.Parallel() + repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace) + knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) defer tearDown(t, c, namespace) - hasSecretConfig, err := CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, "kaniko-secret") - if err != nil { - t.Fatalf("Expected to create kaniko creds: %v", err) + if _, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Create(getRegistryDeployment(namespace)); err != nil { + t.Fatalf("Failed to create the local registry deployment: %v", err) + } + service := getRegistryService(namespace) + if _, err := c.KubeClient.Kube.CoreV1().Services(namespace).Create(service); err != nil { + t.Fatalf("Failed to create the local registry service: %v", err) + } + set := labels.Set(service.Spec.Selector) + if pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil { + t.Fatalf("Failed to list Pods of service[%s] error:%v", service.GetName(), err) + } else { + if len(pods.Items) != 1 { + t.Fatalf("Only 1 pod for service %s should be running: %v", service, pods.Items) + } + + if err := WaitForPodState(c, pods.Items[0].Name, namespace, func(pod *corev1.Pod) (bool, error) { + return pod.Status.Phase == "Running", nil + }, "PodContainersRunning"); err != nil { + t.Fatalf("Error waiting for Pod %q to run: %v", pods.Items[0].Name, err) + } } t.Logf("Creating Git PipelineResource %s", kanikoGitResourceName) @@ -68,7 +86,7 @@ func TestKanikoTaskRun(t *testing.T) { } t.Logf("Creating Task %s", kanikoTaskName) - if _, err := c.TaskClient.Create(getTask(repo, namespace, hasSecretConfig)); err != nil { + if _, err := c.TaskClient.Create(getTask(repo, namespace)); err != nil { t.Fatalf("Failed to create Task `%s`: %s", kanikoTaskName, err) } @@ -111,12 +129,58 @@ func TestKanikoTaskRun(t *testing.T) { } // match the local digest, which is first capture group against the remote image - remoteDigest, err := getRemoteDigest(repo) + remoteDigest, err := getRemoteDigest(t, c, namespace, repo) if err != nil { - t.Fatalf("Expected to get digest for remote image %s", repo) + t.Fatalf("Expected to get digest for remote image %s: %v", repo, err) + } + if d := cmp.Diff(digest, remoteDigest); d != "" { + t.Fatalf("Expected local digest %s to match remote digest %s: %s", digest, remoteDigest, d) + } +} + +func getRegistryDeployment(namespace string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "registry", + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "registry", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "registry", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "registry", + Image: "registry", + }}, + }, + }, + }, } - if digest != remoteDigest { - t.Fatalf("Expected local digest %s to match remote digest %s", digest, remoteDigest) +} + +func getRegistryService(namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "registry", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{ + Port: 5000, + }}, + Selector: map[string]string{ + "app": "registry", + }, + }, } } @@ -135,7 +199,7 @@ func getImageResource(namespace, repo string) *v1alpha1.PipelineResource { )) } -func getTask(repo, namespace string, withSecretConfig bool) *v1alpha1.Task { +func getTask(repo, namespace string) *v1alpha1.Task { taskSpecOps := []tb.TaskSpecOp{ tb.TaskInputs(tb.InputsResource("gitsource", v1alpha1.PipelineResourceTypeGit)), tb.TaskOutputs(tb.OutputsResource("builtImage", v1alpha1.PipelineResourceTypeImage)), @@ -146,21 +210,15 @@ func getTask(repo, namespace string, withSecretConfig bool) *v1alpha1.Task { fmt.Sprintf("--destination=%s", repo), "--context=/workspace/gitsource", "--oci-layout-path=/workspace/output/builtImage", + "--insecure", + "--insecure-pull", + "--insecure-registry=registry"+namespace+":5000/", ), } - if withSecretConfig { - stepOps = append(stepOps, - tb.StepVolumeMount("kaniko-secret", "/secrets"), - tb.StepEnvVar("GOOGLE_APPLICATION_CREDENTIALS", "/secrets/config.json"), - ) - taskSpecOps = append(taskSpecOps, tb.TaskVolume("kaniko-secret", tb.VolumeSource(corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "kaniko-secret", - }, - }))) - } step := tb.Step("kaniko", "gcr.io/kaniko-project/executor:v0.13.0", stepOps...) taskSpecOps = append(taskSpecOps, step) + sidecar := tb.Sidecar("registry", "registry") + taskSpecOps = append(taskSpecOps, sidecar) return tb.Task(kanikoTaskName, namespace, tb.TaskSpec(taskSpecOps...)) } @@ -174,18 +232,34 @@ func getTaskRun(namespace string) *v1alpha1.TaskRun { )) } -func getRemoteDigest(image string) (string, error) { - ref, err := name.ParseReference(image, name.WeakValidation) - if err != nil { - return "", xerrors.Errorf("could not parse image reference %q: %w", image, err) +func getRemoteDigest(t *testing.T, c *clients, namespace, image string) (string, error) { + t.Helper() + podName := "skopeo-jq" + if _, err := c.KubeClient.Kube.CoreV1().Pods(namespace).Create(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: podName, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "skopeo", + Image: "gcr.io/tekton-releases/dogfooding/skopeo:latest", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"skopeo inspect --tls-verify=false docker://" + image + ":latest| jq '.Digest'"}, + }}, + RestartPolicy: corev1.RestartPolicyNever, + }, + }); err != nil { + t.Fatalf("Failed to create the local registry service: %v", err) } - img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) - if err != nil { - return "", xerrors.Errorf("could not pull remote ref %s: %w", ref, err) + if err := WaitForPodState(c, podName, namespace, func(pod *corev1.Pod) (bool, error) { + return pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed", nil + }, "PodContainersTerminated"); err != nil { + t.Fatalf("Error waiting for Pod %q to terminate: %v", podName, err) } - digest, err := img.Digest() + logs, err := getContainerLogsFromPod(c.KubeClient.Kube, podName, "skopeo", namespace) if err != nil { - return "", xerrors.Errorf("could not get digest for image %s: %w", img, err) + t.Fatalf("Could not get logs for pod %s: %s", podName, err) } - return digest.String(), nil + return strings.TrimSpace(strings.ReplaceAll(logs, "\"", "")), nil } From 5dfe45712a08b3fb9afa1dea6997a979095811d9 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 5 Nov 2019 14:58:31 +0100 Subject: [PATCH 2/2] =?UTF-8?q?Refactor=20setup=20to=20take=20extra=20setu?= =?UTF-8?q?p=20functions=20=E2=9B=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And use this to extract the "local" registry part of TestKanikoTaskRun into a separate function, reusable in other tests. Signed-off-by: Vincent Demeester --- test/init_test.go | 7 ++- test/kaniko_task_test.go | 76 ++--------------------------- test/registry_test.go | 102 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 74 deletions(-) create mode 100644 test/registry_test.go diff --git a/test/init_test.go b/test/init_test.go index b5db87fe93e..c8c08cda228 100644 --- a/test/init_test.go +++ b/test/init_test.go @@ -44,7 +44,7 @@ import ( var initMetrics sync.Once -func setup(t *testing.T) (*clients, string) { +func setup(t *testing.T, fn ...func(*testing.T, *clients, string)) (*clients, string) { t.Helper() namespace := names.SimpleNameGenerator.RestrictLengthWithRandomSuffix("arendelle") @@ -53,6 +53,11 @@ func setup(t *testing.T) (*clients, string) { c := newClients(t, knativetest.Flags.Kubeconfig, knativetest.Flags.Cluster, namespace) createNamespace(t, namespace, c.KubeClient) verifyServiceAccountExistence(t, namespace, c.KubeClient) + + for _, f := range fn { + f(t, c, namespace) + } + return c, namespace } diff --git a/test/kaniko_task_test.go b/test/kaniko_task_test.go index 18399a8e14f..d237e7e4a78 100644 --- a/test/kaniko_task_test.go +++ b/test/kaniko_task_test.go @@ -27,10 +27,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" tb "github.com/tektoncd/pipeline/test/builder" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" knativetest "knative.dev/pkg/test" ) @@ -45,7 +43,7 @@ const ( // TestTaskRun is an integration test that will verify a TaskRun using kaniko func TestKanikoTaskRun(t *testing.T) { - c, namespace := setup(t) + c, namespace := setup(t, withRegistry) t.Parallel() repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace) @@ -53,28 +51,6 @@ func TestKanikoTaskRun(t *testing.T) { knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) defer tearDown(t, c, namespace) - if _, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Create(getRegistryDeployment(namespace)); err != nil { - t.Fatalf("Failed to create the local registry deployment: %v", err) - } - service := getRegistryService(namespace) - if _, err := c.KubeClient.Kube.CoreV1().Services(namespace).Create(service); err != nil { - t.Fatalf("Failed to create the local registry service: %v", err) - } - set := labels.Set(service.Spec.Selector) - if pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil { - t.Fatalf("Failed to list Pods of service[%s] error:%v", service.GetName(), err) - } else { - if len(pods.Items) != 1 { - t.Fatalf("Only 1 pod for service %s should be running: %v", service, pods.Items) - } - - if err := WaitForPodState(c, pods.Items[0].Name, namespace, func(pod *corev1.Pod) (bool, error) { - return pod.Status.Phase == "Running", nil - }, "PodContainersRunning"); err != nil { - t.Fatalf("Error waiting for Pod %q to run: %v", pods.Items[0].Name, err) - } - } - t.Logf("Creating Git PipelineResource %s", kanikoGitResourceName) if _, err := c.PipelineResourceClient.Create(getGitResource(namespace)); err != nil { t.Fatalf("Failed to create Pipeline Resource `%s`: %s", kanikoGitResourceName, err) @@ -138,52 +114,6 @@ func TestKanikoTaskRun(t *testing.T) { } } -func getRegistryDeployment(namespace string) *appsv1.Deployment { - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "registry", - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "registry", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "registry", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "registry", - Image: "registry", - }}, - }, - }, - }, - } -} - -func getRegistryService(namespace string) *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "registry", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{{ - Port: 5000, - }}, - Selector: map[string]string{ - "app": "registry", - }, - }, - } -} - func getGitResource(namespace string) *v1alpha1.PipelineResource { return tb.PipelineResource(kanikoGitResourceName, namespace, tb.PipelineResourceSpec( v1alpha1.PipelineResourceTypeGit, @@ -212,7 +142,7 @@ func getTask(repo, namespace string) *v1alpha1.Task { "--oci-layout-path=/workspace/output/builtImage", "--insecure", "--insecure-pull", - "--insecure-registry=registry"+namespace+":5000/", + "--insecure-registry=registry."+namespace+":5000/", ), } step := tb.Step("kaniko", "gcr.io/kaniko-project/executor:v0.13.0", stepOps...) @@ -250,7 +180,7 @@ func getRemoteDigest(t *testing.T, c *clients, namespace, image string) (string, RestartPolicy: corev1.RestartPolicyNever, }, }); err != nil { - t.Fatalf("Failed to create the local registry service: %v", err) + t.Fatalf("Failed to create the skopeo-jq pod: %v", err) } if err := WaitForPodState(c, podName, namespace, func(pod *corev1.Pod) (bool, error) { return pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed", nil diff --git a/test/registry_test.go b/test/registry_test.go new file mode 100644 index 00000000000..58eb638643a --- /dev/null +++ b/test/registry_test.go @@ -0,0 +1,102 @@ +// +build e2e + +/* +Copyright 2019 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package test + +import ( + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func withRegistry(t *testing.T, c *clients, namespace string) { + if _, err := c.KubeClient.Kube.AppsV1().Deployments(namespace).Create(getRegistryDeployment(namespace)); err != nil { + t.Fatalf("Failed to create the local registry deployment: %v", err) + } + service := getRegistryService(namespace) + if _, err := c.KubeClient.Kube.CoreV1().Services(namespace).Create(service); err != nil { + t.Fatalf("Failed to create the local registry service: %v", err) + } + set := labels.Set(service.Spec.Selector) + + // Give it a little bit of time to at least create the pod + time.Sleep(5 * time.Second) + + if pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil { + t.Fatalf("Failed to list Pods of service[%s] error:%v", service.GetName(), err) + } else { + if len(pods.Items) != 1 { + t.Fatalf("Only 1 pod for service %s should be running: %v", service, pods.Items) + } + + if err := WaitForPodState(c, pods.Items[0].Name, namespace, func(pod *corev1.Pod) (bool, error) { + return pod.Status.Phase == "Running", nil + }, "PodContainersRunning"); err != nil { + t.Fatalf("Error waiting for Pod %q to run: %v", pods.Items[0].Name, err) + } + } +} + +func getRegistryDeployment(namespace string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "registry", + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "registry", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "registry", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "registry", + Image: "registry", + }}, + }, + }, + }, + } +} + +func getRegistryService(namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "registry", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{ + Port: 5000, + }}, + Selector: map[string]string{ + "app": "registry", + }, + }, + } +}