From 3fdf65e0ba9e0c3ce6b69da94925c2e1d620a614 Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 09:57:14 +0000 Subject: [PATCH 1/8] wait longer for rollouts to complete --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6a86706..8524aff 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ smokey_agent_install: $(maybe_docker_build) helm repo add honeycomb https://honeycombio.github.io/helm-charts OTEL_COLLECTOR_IP="$(call get_collector_ip)" \ envsubst < smoke-tests/agent-helm-values.yaml | helm install smokey-agent honeycomb/network-agent --values - - kubectl rollout status daemonset.apps/smokey-agent-network-agent --timeout=10s + kubectl rollout status daemonset.apps/smokey-agent-network-agent --timeout=60s smokey_copy_output: # copy output from collector file to local machine @@ -69,7 +69,7 @@ get_collector_ip = \ smokey_echo_job: make apply-echoserver kubectl create --filename smoke-tests/smoke-job.yaml - kubectl rollout status deployment.apps/echoserver --timeout=10s --namespace echoserver + kubectl rollout status deployment.apps/echoserver --timeout=60s --namespace echoserver kubectl wait --for=condition=complete job/smoke-job --timeout=60s --namespace echoserver .PHONY: unsmoke From 94ad2bddcb994630693bea0a97ea8faebdc54f57 Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 09:57:29 +0000 Subject: [PATCH 2/8] reduce number of echoservers from 5 to 1 --- smoke-tests/echoserver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoke-tests/echoserver.yaml b/smoke-tests/echoserver.yaml index 9f3ade1..9ce0e24 100644 --- a/smoke-tests/echoserver.yaml +++ b/smoke-tests/echoserver.yaml @@ -9,7 +9,7 @@ metadata: name: echoserver namespace: echoserver spec: - replicas: 5 + replicas: 1 selector: matchLabels: app: echoserver From e91212138165e75bd5ebc05eaf47fd35712e28f4 Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 10:21:24 +0000 Subject: [PATCH 3/8] attempt to get collector output file a few times --- Makefile | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 8524aff..50bd71c 100644 --- a/Makefile +++ b/Makefile @@ -49,12 +49,20 @@ smokey_agent_install: $(maybe_docker_build) kubectl rollout status daemonset.apps/smokey-agent-network-agent --timeout=60s smokey_copy_output: - # copy output from collector file to local machine - # this is ignored in .gitignore - # TODO use more reliable way to wait for file to be ready - # for now just sleeping a bit - sleep 12 - kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json + # removes the file first in case it already exists + if [ -f ./smoke-tests/traces-orig.json ]; then \ + rm ./smoke-tests/traces-orig.json; \ + fi + # copy collector output file to local machine to run tests on + # the file may not be ready immediately, so retry a few times + # note: the file is ignored in .gitignore + for i in {1..10}; do \ + kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \ + if [ -f ./smoke-tests/traces-orig.json ]; then \ + break; \ + fi; \ + sleep 5; \ + done smokey_verify_output: # verify that the output from the collector matches the expected output From 47cf2a827bd4499e4991517f45766f3a5d19c659 Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 11:38:19 +0000 Subject: [PATCH 4/8] move delay to begining of loop --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 50bd71c..99ced0b 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,11 @@ smokey_copy_output: # the file may not be ready immediately, so retry a few times # note: the file is ignored in .gitignore for i in {1..10}; do \ + sleep 10; \ kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \ if [ -f ./smoke-tests/traces-orig.json ]; then \ break; \ fi; \ - sleep 5; \ done smokey_verify_output: From 29e51951e186631d06c756dd63b55a478e959326 Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 12:09:25 +0000 Subject: [PATCH 5/8] use -s to check if the file is empty --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 99ced0b..03622f4 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ smokey_copy_output: for i in {1..10}; do \ sleep 10; \ kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \ - if [ -f ./smoke-tests/traces-orig.json ]; then \ + if [ -s ./smoke-tests/traces-orig.json ]; then \ break; \ fi; \ done From 69b1eab8741e12653d9956fc944dec6ca2f89f71 Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 12:22:02 +0000 Subject: [PATCH 6/8] add logs --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 03622f4..5772a6e 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,13 @@ smokey_copy_output: # the file may not be ready immediately, so retry a few times # note: the file is ignored in .gitignore for i in {1..10}; do \ - sleep 10; \ + echo "attempt $$i to get collector out file"; \ kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \ if [ -s ./smoke-tests/traces-orig.json ]; then \ + echo "got collector out file"; \ break; \ fi; \ + sleep 10; \ done smokey_verify_output: From 7127e278d48f4248ad59e3f11a241c00768463cc Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Thu, 30 Nov 2023 12:37:37 +0000 Subject: [PATCH 7/8] remove loop and wait longer --- Makefile | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 5772a6e..3b1a22e 100644 --- a/Makefile +++ b/Makefile @@ -56,15 +56,11 @@ smokey_copy_output: # copy collector output file to local machine to run tests on # the file may not be ready immediately, so retry a few times # note: the file is ignored in .gitignore - for i in {1..10}; do \ - echo "attempt $$i to get collector out file"; \ - kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \ - if [ -s ./smoke-tests/traces-orig.json ]; then \ - echo "got collector out file"; \ - break; \ - fi; \ - sleep 10; \ - done + sleep 30 + kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \ + if [ ! -s ./smoke-tests/traces-orig.json ]; then \ + echo "failed to copy collector output file"; \ + fi; smokey_verify_output: # verify that the output from the collector matches the expected output From 66386110027150a1762534b2b54e61d43f0426db Mon Sep 17 00:00:00 2001 From: Mike Goldsmth Date: Fri, 1 Dec 2023 15:57:28 +0000 Subject: [PATCH 8/8] update comment to be more accurate about waiting --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3b1a22e..ab2d3f8 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ smokey_copy_output: rm ./smoke-tests/traces-orig.json; \ fi # copy collector output file to local machine to run tests on - # the file may not be ready immediately, so retry a few times + # the file may not be ready immediately, so wait before trying to copy # note: the file is ignored in .gitignore sleep 30 kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json; \