From 97e758ef3d8637678018ddecf28738863bb4b799 Mon Sep 17 00:00:00 2001 From: Nathan Cutler Date: Wed, 22 Apr 2020 22:11:41 +0200 Subject: [PATCH] provision.sh: refactor PROVISIONED_NODES while loop If all the "/tmp/ready-{{ node.name }}" files don't get created in the first iteration of the loop, the scp fails and that causes the whole script to fail because it is running under "set -e". (This while loop was written before we had "set -e" at the top of the script.) Since we're touching the code of the loop anyway, take this opportunity to refactor it. Fixes: c0d5df13b20635ad8a7a0075f74b21800cd10ffc Fixes: https://github.com/SUSE/sesdev/issues/278 Signed-off-by: Nathan Cutler --- contrib/standalone.sh | 12 +++---- sesdev.spec | 1 - .../ceph-salt/ceph_salt_deployment.sh.j2 | 4 +-- .../deepsea/deepsea_deployment.sh.j2 | 4 +-- .../templates/deepsea/ses5_pre_stage_0.sh.j2 | 2 +- seslib/templates/provision.sh.j2 | 36 +++++++++++++------ .../templates/{qa_test.sh => qa_test.sh.j2} | 0 seslib/templates/wait_for_minions.sh | 21 ----------- seslib/templates/wait_for_minions.sh.j2 | 19 ++++++++++ 9 files changed, 55 insertions(+), 44 deletions(-) rename seslib/templates/{qa_test.sh => qa_test.sh.j2} (100%) delete mode 100644 seslib/templates/wait_for_minions.sh create mode 100644 seslib/templates/wait_for_minions.sh.j2 diff --git a/contrib/standalone.sh b/contrib/standalone.sh index 7e983316..f4a433e3 100755 --- a/contrib/standalone.sh +++ b/contrib/standalone.sh @@ -171,7 +171,7 @@ if [ "$(sesdev list --format json | jq -r '. | length')" != "0" ] ; then fi if [ "$SES5" ] ; then - run_cmd sesdev box remove --non-interactive sles-12-sp3 || true + sesdev box remove --non-interactive sles-12-sp3 # deploy ses5 without igw, so as not to hit https://github.com/SUSE/sesdev/issues/239 run_cmd sesdev create ses5 --non-interactive --roles "[master,storage,mon,mgr,mds,rgw,nfs]" --qa-test ses5-1node run_cmd sesdev destroy --non-interactive ses5-1node @@ -184,7 +184,7 @@ if [ "$SES5" ] ; then fi if [ "$NAUTILUS" ] ; then - run_cmd sesdev box remove --non-interactive leap-15.1 || true + sesdev box remove --non-interactive leap-15.1 run_cmd sesdev create nautilus --non-interactive --single-node --qa-test nautilus-1node run_cmd sesdev destroy --non-interactive nautilus-1node run_cmd sesdev create nautilus --non-interactive nautilus-4node @@ -193,7 +193,7 @@ if [ "$NAUTILUS" ] ; then fi if [ "$SES6" ] ; then - run_cmd sesdev box remove --non-interactive sles-15-sp1 || true + sesdev box remove --non-interactive sles-15-sp1 run_cmd sesdev create ses6 --non-interactive --single-node --qa-test ses6-1node run_cmd sesdev destroy --non-interactive ses6-1node run_cmd sesdev create ses6 --non-interactive ses6-4node @@ -205,7 +205,7 @@ if [ "$SES6" ] ; then fi if [ "$OCTOPUS" ] ; then - run_cmd sesdev box remove --non-interactive leap-15.2 || true + sesdev box remove --non-interactive leap-15.2 run_cmd sesdev create octopus --non-interactive $CEPH_SALT_FROM_SOURCE --single-node --qa-test octopus-1node run_cmd sesdev destroy --non-interactive octopus-1node run_cmd sesdev create octopus --non-interactive $CEPH_SALT_FROM_SOURCE octopus-4node @@ -214,7 +214,7 @@ if [ "$OCTOPUS" ] ; then fi if [ "$SES7" ] ; then - run_cmd sesdev box remove --non-interactive sles-15-sp2 || true + sesdev box remove --non-interactive sles-15-sp2 run_cmd sesdev create ses7 --non-interactive $CEPH_SALT_FROM_SOURCE --single-node --qa-test ses7-1node run_cmd sesdev destroy --non-interactive ses7-1node run_cmd sesdev create ses7 --non-interactive $CEPH_SALT_FROM_SOURCE ses7-4node @@ -226,7 +226,7 @@ if [ "$SES7" ] ; then fi if [ "$PACIFIC" ] ; then - run_cmd sesdev box remove --non-interactive leap-15.2 || true + sesdev box remove --non-interactive leap-15.2 run_cmd sesdev create pacific --non-interactive $CEPH_SALT_FROM_SOURCE --single-node --qa-test pacific-1node run_cmd sesdev destroy --non-interactive pacific-1node run_cmd sesdev create pacific --non-interactive $CEPH_SALT_FROM_SOURCE pacific-4node diff --git a/sesdev.spec b/sesdev.spec index 049ca459..45d3305b 100644 --- a/sesdev.spec +++ b/sesdev.spec @@ -82,7 +82,6 @@ sed -i -e 's/^\s*lv.qemu_use_session = false$//g' seslib/templates/Vagrantfile.j %install %py3_install %fdupes %{buildroot}%{python3_sitelib} -# qa script installation install -m 0755 -d %{buildroot}/%{_datadir}/%{name}/qa install -m 0755 -d %{buildroot}/%{_datadir}/%{name}/qa/common install -m 0755 qa/health-ok.sh %{buildroot}/%{_datadir}/%{name}/qa/health-ok.sh diff --git a/seslib/templates/ceph-salt/ceph_salt_deployment.sh.j2 b/seslib/templates/ceph-salt/ceph_salt_deployment.sh.j2 index 7bff8f3e..5312b895 100644 --- a/seslib/templates/ceph-salt/ceph_salt_deployment.sh.j2 +++ b/seslib/templates/ceph-salt/ceph_salt_deployment.sh.j2 @@ -29,7 +29,7 @@ zypper -n in ceph-salt {% endif %} systemctl restart salt-master -{% include "wait_for_minions.sh" %} +{% include "wait_for_minions.sh.j2" %} {% if stop_before_ceph_salt_config %} exit 0 @@ -132,4 +132,4 @@ echo "{\"service_type\": \"osd\", \"placement\": {\"host_pattern\": \"{{ node.na ceph fs volume create myfs "$MDS_NODES_COMMA_SEPARATED_LIST" {% endif %} {# if ceph_salt_deploy_mdss #} -{% include "qa_test.sh" %} +{% include "qa_test.sh.j2" %} diff --git a/seslib/templates/deepsea/deepsea_deployment.sh.j2 b/seslib/templates/deepsea/deepsea_deployment.sh.j2 index 7d5bd868..5abc420f 100644 --- a/seslib/templates/deepsea/deepsea_deployment.sh.j2 +++ b/seslib/templates/deepsea/deepsea_deployment.sh.j2 @@ -23,7 +23,7 @@ EOF chown -R salt:salt /srv -{% include "wait_for_minions.sh" %} +{% include "wait_for_minions.sh.j2" %} sleep 2 salt '*' grains.set deepsea True @@ -143,4 +143,4 @@ sleep 5 echo "deployment complete!" -{% include "qa_test.sh" %} +{% include "qa_test.sh.j2" %} diff --git a/seslib/templates/deepsea/ses5_pre_stage_0.sh.j2 b/seslib/templates/deepsea/ses5_pre_stage_0.sh.j2 index 7a1c2cc8..b152f950 100644 --- a/seslib/templates/deepsea/ses5_pre_stage_0.sh.j2 +++ b/seslib/templates/deepsea/ses5_pre_stage_0.sh.j2 @@ -12,7 +12,7 @@ SuSEfirewall2 off sed -i 's/#worker_threads: 5/worker_threads: 10/g' /etc/salt/master systemctl restart salt-master systemctl restart salt-minion -{% include "wait_for_minions.sh" %} +{% include "wait_for_minions.sh.j2" %} cat > /srv/salt/ceph/updates/nop.sls </dev/null | wc -l` - echo "waiting for nodes (${PROVISIONED_NODES}/{{ nodes|length }})"; - [[ "${PROVISIONED_NODES}" != "{{ nodes|length }}" ]] || break - sleep 2; +set +x +rm -f /tmp/ready-* +echo "Waiting for {{ nodes | length }} nodes to complete initial provisioning..." +while true ; do {% for node in nodes %} - scp -o StrictHostKeyChecking=no {{ node.name }}:/tmp/ready /tmp/ready-{{ node.name }}; + scp -o StrictHostKeyChecking=no {{ node.name }}:/tmp/ready /tmp/ready-{{ node.name }} || true {% endfor %} + PROVISIONED_NODES="$(find /tmp -maxdepth 1 -type f -regex '/tmp/ready-.*' | wc --lines)" + echo "${PROVISIONED_NODES} of {{ nodes | length }} have completed initial provisioning." + [ "${PROVISIONED_NODES}" = "{{ nodes|length }}" ] && break + set -x + sleep 3 + set +x done +set -x {% if node == master %} +set +x +echo "Waiting for {{ nodes | length }} minions to submit their keys..." while true; do - sleep 1 + set -x salt-key -L - N=`salt-key --out=json -L | jq '.minions_pre | length'` - [ "$N" == "{{ nodes|length }}" ] && break + set +x + N=$(salt-key --out=json -L | jq '.minions_pre | length') + echo "${N} of {{ nodes | length }} minions have submitted their keys." + [ "$N" = "{{ nodes|length }}" ] && break + set -x + sleep 3 + set +x done +set -x salt-key -Ay -{% include "wait_for_minions.sh" %} +{% include "wait_for_minions.sh.j2" %} {% endif %} {# node == master #} {% if deployment_tool == "deepsea" %} diff --git a/seslib/templates/qa_test.sh b/seslib/templates/qa_test.sh.j2 similarity index 100% rename from seslib/templates/qa_test.sh rename to seslib/templates/qa_test.sh.j2 diff --git a/seslib/templates/wait_for_minions.sh b/seslib/templates/wait_for_minions.sh deleted file mode 100644 index 588b6c87..00000000 --- a/seslib/templates/wait_for_minions.sh +++ /dev/null @@ -1,21 +0,0 @@ - -# make sure all minions are responding -set +ex -LOOP_COUNT="0" -while true ; do - set -x - sleep 5 - set +x - if [ "$LOOP_COUNT" -ge "20" ] ; then - echo "ERROR: minion(s) not responding to ping?" - exit 1 - fi - LOOP_COUNT="$((LOOP_COUNT + 1))" - set -x - MINIONS_RESPONDING="$(salt '*' test.ping | grep True | wc --lines)" - if [ "$MINIONS_RESPONDING" = "{{ nodes|length }}" ]; then - break - fi - set +x -done -set -ex diff --git a/seslib/templates/wait_for_minions.sh.j2 b/seslib/templates/wait_for_minions.sh.j2 new file mode 100644 index 00000000..663dbf1f --- /dev/null +++ b/seslib/templates/wait_for_minions.sh.j2 @@ -0,0 +1,19 @@ + +# make sure all minions are responding +set +ex +LOOP_COUNT="0" +while true ; do + if [ "$LOOP_COUNT" -ge "30" ] ; then + echo "ERROR: minion(s) not responding to ping?" + exit 1 + fi + LOOP_COUNT="$((LOOP_COUNT + 1))" + echo "Pinging {{ nodes | length }} minions..." + MINIONS_RESPONDING="$(salt '*' test.ping 2> /dev/null | grep True | wc --lines)" + echo "${MINIONS_RESPONDING} of {{ nodes | length }} minions responded to ping." + [ "$MINIONS_RESPONDING" = "{{ nodes|length }}" ] && break + set -x + sleep 3 + set +x +done +set -ex