From f485c14cf8c15ae3d1d098f6b6ccbc698cf21db1 Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Tue, 5 Feb 2019 15:22:28 +0100 Subject: [PATCH 01/10] Elasticsearch tests --- k8s/elasticsearch/apptest/tester/Dockerfile | 1 + .../apptest/tester/tests/basic-suite.yaml | 95 ++++++++++++++++++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/k8s/elasticsearch/apptest/tester/Dockerfile b/k8s/elasticsearch/apptest/tester/Dockerfile index 960bcff7f9..50d8c1871f 100644 --- a/k8s/elasticsearch/apptest/tester/Dockerfile +++ b/k8s/elasticsearch/apptest/tester/Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ jq \ uuid-runtime \ wget \ + curl \ && rm -rf /var/lib/apt/lists/* RUN wget -q -O /bin/kubectl \ diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index a2752fb671..f6b5956954 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -1,8 +1,97 @@ actions: -- name: dummy test +- name: Wait until all expected nodes are connected to the cluster bashTest: script: |- - exit 0 + function get_nodes_count() { + local readonly health_url="$1" + nodes_count="$(curl -s "${health_url}" | jq '.number_of_nodes')" + echo "${nodes_count}" + } + + expected_nodes="${REPLICAS}" + + timeout=300 + health_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200/_cluster/health" + timeout_time=$(( $(date +%s) + timeout )) + + # Wait for all nodes to join the cluster + nodes_count="$(get_nodes_count "${health_url}")" + while [[ "${nodes_count}" -ne "${expected_nodes}" ]] \ + && [[ "$(date +%s)" -le "${timeout_time}" ]]; do + sleep 3 + nodes_count="$(get_nodes_count "${health_url}")" + done + + if [[ "${nodes_count}" -eq "${expected_nodes}" ]]; then + echo "OK - all nodes joined" + else + echo "FAIL - nodes count: ${nodes_count}" + fi expect: - statusCode: + stdout: + contains: OK - all nodes joined + +- name: Wait for green status in the cluster + bashTest: + script: |- + function get_cluster_status() { + local readonly health_url="$1" + status="$(curl -s "${health_url}" | jq '.status')" + echo "${status}" + } + + timeout=300 + health_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200/_cluster/health" + + timeout_time=$(( $(date +%s) + timeout )) + + # Wait for green status + status="$(get_cluster_status "${health_url}")" + while [[ "${status}" != '"green"' ]] \ + && [[ "$(date +%s)" -le "${timeout_time}" ]]; do + sleep 3 + status="$(get_cluster_status "${health_url}")" + done + + if [[ "${status}" == '"green"' ]]; then + echo "OK - green status" + else + echo "FAIL - status: ${status}" + fi + expect: + stdout: + contains: OK - green status + +- name: Add a document + bashTest: + script: |- + elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" + curl -s -f -X PUT -H 'Content-Type:application/json' -d '{ + "name": "John", + "surname": "Smith" + }' "${elastic_url}/employees/person/1" + expect: + exitCode: equals: 0 + +- name: Update the document + bashTest: + script: |- + elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" + curl -s -f -X POST -H 'Content-Type:application/json' -d '{ + "doc": { + "lastname": "Jones" + } + }' "${elastic_url}/employees/person/1/_update" + expect: + exitCode: + equals: 0 + +- name: Search for the document + bashTest: + script: |- + elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" + curl -s -f "${elastic_url}/_search?q=jones" + expect: + stdout: + contains: "\"name\":\"John\"" \ No newline at end of file From 10b9ad3758d6ad9b844966ae01cebc65e49b59db Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Wed, 6 Feb 2019 09:55:11 +0100 Subject: [PATCH 02/10] Increase the timeout --- k8s/elasticsearch/deployer/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/elasticsearch/deployer/Dockerfile b/k8s/elasticsearch/deployer/Dockerfile index d4ca281276..d6d45b24ea 100644 --- a/k8s/elasticsearch/deployer/Dockerfile +++ b/k8s/elasticsearch/deployer/Dockerfile @@ -8,6 +8,10 @@ COPY apptest/deployer /data-test/ # Provide registry prefix and tag for default values for images. ARG REGISTRY ARG TAG + +ENV WAIT_FOR_READY_TIMEOUT 600 +ENV TESTER_TIMEOUT 600 + RUN cat /data/schema.yaml \ | env -i "REGISTRY=$REGISTRY" "TAG=$TAG" envsubst \ > /data/schema.yaml.new \ From f9701a2690cf03252dbcd3a55bba6b234adbe159 Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Wed, 6 Feb 2019 11:05:54 +0100 Subject: [PATCH 03/10] Wait for object propagation in ES test --- k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index f6b5956954..f3ec1eeb8c 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -90,6 +90,8 @@ actions: - name: Search for the document bashTest: script: |- + # Wait to allow the object to be propagated between all nodes + sleep 5 elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" curl -s -f "${elastic_url}/_search?q=jones" expect: From 772b78abf10edd0f663095b6c5d1a1fe2d4d043d Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Wed, 6 Feb 2019 16:07:55 +0100 Subject: [PATCH 04/10] Add replicas to --- .../apptest/deployer/manifest/tester.yaml.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template b/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template index 86d6bfe7ca..7f0c290d06 100644 --- a/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template +++ b/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template @@ -27,4 +27,6 @@ spec: fieldPath: metadata.namespace - name: APP_INSTANCE_NAME value: ${APP_INSTANCE_NAME} + - name: REPLICAS + value: ${REPLICAS} restartPolicy: Never From 6e49f07e3907f6158e7dd13b096c019b88e4faf1 Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Wed, 6 Feb 2019 16:23:08 +0100 Subject: [PATCH 05/10] Refactoring --- .../deployer/manifest/tester.yaml.template | 4 +-- k8s/elasticsearch/apptest/tester/tester.sh | 3 ++ .../apptest/tester/tests/basic-suite.yaml | 28 ++++++------------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template b/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template index 7f0c290d06..6fad6599d2 100644 --- a/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template +++ b/k8s/elasticsearch/apptest/deployer/manifest/tester.yaml.template @@ -26,7 +26,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: APP_INSTANCE_NAME - value: ${APP_INSTANCE_NAME} + value: "${APP_INSTANCE_NAME}" - name: REPLICAS - value: ${REPLICAS} + value: "${REPLICAS}" restartPolicy: Never diff --git a/k8s/elasticsearch/apptest/tester/tester.sh b/k8s/elasticsearch/apptest/tester/tester.sh index ddccd4cf33..cb73886e0f 100755 --- a/k8s/elasticsearch/apptest/tester/tester.sh +++ b/k8s/elasticsearch/apptest/tester/tester.sh @@ -17,6 +17,9 @@ set -xeo pipefail shopt -s nullglob +export ELASTIC_URL="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" +export HEALTH_URL="${ELASTIC_URL}/_cluster/health" + for test in /tests/*; do testrunner -logtostderr "--test_spec=${test}" done diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index f3ec1eeb8c..c3ac05df57 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -3,23 +3,20 @@ actions: bashTest: script: |- function get_nodes_count() { - local readonly health_url="$1" - nodes_count="$(curl -s "${health_url}" | jq '.number_of_nodes')" - echo "${nodes_count}" + curl -s "${HEALTH_URL}" | jq '.number_of_nodes' } expected_nodes="${REPLICAS}" timeout=300 - health_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200/_cluster/health" timeout_time=$(( $(date +%s) + timeout )) # Wait for all nodes to join the cluster - nodes_count="$(get_nodes_count "${health_url}")" + nodes_count="$(get_nodes_count)" while [[ "${nodes_count}" -ne "${expected_nodes}" ]] \ && [[ "$(date +%s)" -le "${timeout_time}" ]]; do sleep 3 - nodes_count="$(get_nodes_count "${health_url}")" + nodes_count="$(get_nodes_count)" done if [[ "${nodes_count}" -eq "${expected_nodes}" ]]; then @@ -35,22 +32,18 @@ actions: bashTest: script: |- function get_cluster_status() { - local readonly health_url="$1" - status="$(curl -s "${health_url}" | jq '.status')" - echo "${status}" + curl -s "${HEALTH_URL}" | jq '.status' } timeout=300 - health_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200/_cluster/health" - timeout_time=$(( $(date +%s) + timeout )) # Wait for green status - status="$(get_cluster_status "${health_url}")" + status="$(get_cluster_status)" while [[ "${status}" != '"green"' ]] \ && [[ "$(date +%s)" -le "${timeout_time}" ]]; do sleep 3 - status="$(get_cluster_status "${health_url}")" + status="$(get_cluster_status)" done if [[ "${status}" == '"green"' ]]; then @@ -65,11 +58,10 @@ actions: - name: Add a document bashTest: script: |- - elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" curl -s -f -X PUT -H 'Content-Type:application/json' -d '{ "name": "John", "surname": "Smith" - }' "${elastic_url}/employees/person/1" + }' "${ELASTIC_URL}/employees/person/1" expect: exitCode: equals: 0 @@ -77,12 +69,11 @@ actions: - name: Update the document bashTest: script: |- - elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" curl -s -f -X POST -H 'Content-Type:application/json' -d '{ "doc": { "lastname": "Jones" } - }' "${elastic_url}/employees/person/1/_update" + }' "${ELASTIC_URL}/employees/person/1/_update" expect: exitCode: equals: 0 @@ -92,8 +83,7 @@ actions: script: |- # Wait to allow the object to be propagated between all nodes sleep 5 - elastic_url="http://${APP_INSTANCE_NAME}-elasticsearch-svc:9200" - curl -s -f "${elastic_url}/_search?q=jones" + curl -s -f "${ELASTIC_URL}/_search?q=jones" expect: stdout: contains: "\"name\":\"John\"" \ No newline at end of file From d4118b4f07d554289dfb989c09546a637fe58cb3 Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Wed, 6 Feb 2019 21:11:24 +0100 Subject: [PATCH 06/10] remove ugly " and EOF --- k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index c3ac05df57..6404cd3803 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -32,7 +32,7 @@ actions: bashTest: script: |- function get_cluster_status() { - curl -s "${HEALTH_URL}" | jq '.status' + curl -s "${HEALTH_URL}" | jq -r '.status' } timeout=300 @@ -40,13 +40,13 @@ actions: # Wait for green status status="$(get_cluster_status)" - while [[ "${status}" != '"green"' ]] \ + while [[ "${status}" != 'green' ]] \ && [[ "$(date +%s)" -le "${timeout_time}" ]]; do sleep 3 status="$(get_cluster_status)" done - if [[ "${status}" == '"green"' ]]; then + if [[ "${status}" == 'green' ]]; then echo "OK - green status" else echo "FAIL - status: ${status}" @@ -86,4 +86,4 @@ actions: curl -s -f "${ELASTIC_URL}/_search?q=jones" expect: stdout: - contains: "\"name\":\"John\"" \ No newline at end of file + contains: "\"name\":\"John\"" From 076742e3b8245b68ff971150b66880d048d0c4b8 Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Wed, 6 Feb 2019 21:15:03 +0100 Subject: [PATCH 07/10] Rename actions --- .../apptest/tester/tests/basic-suite.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index 6404cd3803..a5e197d7b7 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -1,5 +1,5 @@ actions: -- name: Wait until all expected nodes are connected to the cluster +- name: All nodes are connected to the cluster bashTest: script: |- function get_nodes_count() { @@ -28,7 +28,7 @@ actions: stdout: contains: OK - all nodes joined -- name: Wait for green status in the cluster +- name: Cluster reports green status bashTest: script: |- function get_cluster_status() { @@ -55,7 +55,7 @@ actions: stdout: contains: OK - green status -- name: Add a document +- name: New document can be saved bashTest: script: |- curl -s -f -X PUT -H 'Content-Type:application/json' -d '{ @@ -66,7 +66,7 @@ actions: exitCode: equals: 0 -- name: Update the document +- name: Document can be updated bashTest: script: |- curl -s -f -X POST -H 'Content-Type:application/json' -d '{ @@ -78,7 +78,7 @@ actions: exitCode: equals: 0 -- name: Search for the document +- name: Existing document can be found bashTest: script: |- # Wait to allow the object to be propagated between all nodes From 128cdefe36235825723cf23bb80cf2be248c7112 Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Fri, 8 Feb 2019 10:07:17 +0100 Subject: [PATCH 08/10] Update basic-suite.yaml --- .../apptest/tester/tests/basic-suite.yaml | 72 +++++++------------ 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index a5e197d7b7..abd497ede5 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -1,59 +1,33 @@ actions: -- name: All nodes are connected to the cluster + +- name: Wait until Elasticsearch is ready bashTest: script: |- - function get_nodes_count() { - curl -s "${HEALTH_URL}" | jq '.number_of_nodes' - } - - expected_nodes="${REPLICAS}" - - timeout=300 - timeout_time=$(( $(date +%s) + timeout )) - - # Wait for all nodes to join the cluster - nodes_count="$(get_nodes_count)" - while [[ "${nodes_count}" -ne "${expected_nodes}" ]] \ - && [[ "$(date +%s)" -le "${timeout_time}" ]]; do - sleep 3 - nodes_count="$(get_nodes_count)" + for ((pod=0; pod<${REPLICAS}; pod++)); do + kubectl wait pod/${APP_INSTANCE_NAME}-elasticsearch-${pod} \ + --namespace=${NAMESPACE} \ + --for=condition=Ready \ + --timeout=180s done + expect: + exitCode: + equals: 0 - if [[ "${nodes_count}" -eq "${expected_nodes}" ]]; then - echo "OK - all nodes joined" - else - echo "FAIL - nodes count: ${nodes_count}" - fi +- name: All nodes are connected to the cluster + bashTest: + script: |- + curl -XGET "${HEALTH_URL}?wait_for_nodes=${REPLICAS}&timeout=300s" | jq '.number_of_nodes' expect: stdout: - contains: OK - all nodes joined + equals: 2 - name: Cluster reports green status bashTest: script: |- - function get_cluster_status() { - curl -s "${HEALTH_URL}" | jq -r '.status' - } - - timeout=300 - timeout_time=$(( $(date +%s) + timeout )) - - # Wait for green status - status="$(get_cluster_status)" - while [[ "${status}" != 'green' ]] \ - && [[ "$(date +%s)" -le "${timeout_time}" ]]; do - sleep 3 - status="$(get_cluster_status)" - done - - if [[ "${status}" == 'green' ]]; then - echo "OK - green status" - else - echo "FAIL - status: ${status}" - fi + curl -XGET "${HEALTH_URL}?wait_for_status=green&timeout=300s" | jq -r '.status' expect: stdout: - contains: OK - green status + equals: 'green' - name: New document can be saved bashTest: @@ -61,10 +35,12 @@ actions: curl -s -f -X PUT -H 'Content-Type:application/json' -d '{ "name": "John", "surname": "Smith" - }' "${ELASTIC_URL}/employees/person/1" + }' "${ELASTIC_URL}/employees/person/1" | jq -r '.result' expect: exitCode: equals: 0 + stdout: + equals: 'created' - name: Document can be updated bashTest: @@ -73,17 +49,19 @@ actions: "doc": { "lastname": "Jones" } - }' "${ELASTIC_URL}/employees/person/1/_update" + }' "${ELASTIC_URL}/employees/person/1/_update" | jq -r '.result' expect: exitCode: equals: 0 + stdout: + equals: 'updated' - name: Existing document can be found bashTest: script: |- # Wait to allow the object to be propagated between all nodes sleep 5 - curl -s -f "${ELASTIC_URL}/_search?q=jones" + curl -s -f "${ELASTIC_URL}/_search?q=name:John" | jq -r '.hits.hits[]._source.lastname' expect: stdout: - contains: "\"name\":\"John\"" + equals: 'Jones' From 53a8054cc3aea14dcb40e634d9bfafe1239ba8cf Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Fri, 8 Feb 2019 10:12:18 +0100 Subject: [PATCH 09/10] Update basic-suite.yaml --- k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index abd497ede5..4402cc7456 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -16,10 +16,15 @@ actions: - name: All nodes are connected to the cluster bashTest: script: |- - curl -XGET "${HEALTH_URL}?wait_for_nodes=${REPLICAS}&timeout=300s" | jq '.number_of_nodes' + nodes="$(curl -XGET "${HEALTH_URL}?wait_for_nodes=${REPLICAS}&timeout=300s" | jq '.number_of_nodes')" + if [[ "${nodes}" -eq "${REPLICAS}" ]]; then + echo "OK - all nodes found" + else + echo "FAIL - found nodes: ${nodes}" + fi expect: stdout: - equals: 2 + equals: OK - all nodes found - name: Cluster reports green status bashTest: From 07956dc73780d6c14ba344ae3d00b20c7bb4664b Mon Sep 17 00:00:00 2001 From: Kamil Hajduczenia Date: Fri, 8 Feb 2019 10:14:03 +0100 Subject: [PATCH 10/10] Update basic-suite.yaml --- k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml index 4402cc7456..615bff02f7 100644 --- a/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml +++ b/k8s/elasticsearch/apptest/tester/tests/basic-suite.yaml @@ -18,13 +18,13 @@ actions: script: |- nodes="$(curl -XGET "${HEALTH_URL}?wait_for_nodes=${REPLICAS}&timeout=300s" | jq '.number_of_nodes')" if [[ "${nodes}" -eq "${REPLICAS}" ]]; then - echo "OK - all nodes found" + echo 'OK - all nodes found' else echo "FAIL - found nodes: ${nodes}" fi expect: stdout: - equals: OK - all nodes found + equals: 'OK - all nodes found' - name: Cluster reports green status bashTest: