Skip to content

Commit

Permalink
Merge pull request #279 from smithfarm/wip-278
Browse files Browse the repository at this point in the history
provision.sh: refactor PROVISIONED_NODES while loop
  • Loading branch information
smithfarm authored Apr 23, 2020
2 parents 204cc67 + 97e758e commit 1ef4e80
Show file tree
Hide file tree
Showing 9 changed files with 55 additions and 44 deletions.
12 changes: 6 additions & 6 deletions contrib/standalone.sh
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ if [ "$(sesdev list --format json | jq -r '. | length')" != "0" ] ; then
fi

if [ "$SES5" ] ; then
run_cmd sesdev box remove --non-interactive sles-12-sp3 || true
sesdev box remove --non-interactive sles-12-sp3
# deploy ses5 without igw, so as not to hit https://github.com/SUSE/sesdev/issues/239
run_cmd sesdev create ses5 --non-interactive --roles "[master,storage,mon,mgr,mds,rgw,nfs]" --qa-test ses5-1node
run_cmd sesdev destroy --non-interactive ses5-1node
Expand All @@ -184,7 +184,7 @@ if [ "$SES5" ] ; then
fi

if [ "$NAUTILUS" ] ; then
run_cmd sesdev box remove --non-interactive leap-15.1 || true
sesdev box remove --non-interactive leap-15.1
run_cmd sesdev create nautilus --non-interactive --single-node --qa-test nautilus-1node
run_cmd sesdev destroy --non-interactive nautilus-1node
run_cmd sesdev create nautilus --non-interactive nautilus-4node
Expand All @@ -193,7 +193,7 @@ if [ "$NAUTILUS" ] ; then
fi

if [ "$SES6" ] ; then
run_cmd sesdev box remove --non-interactive sles-15-sp1 || true
sesdev box remove --non-interactive sles-15-sp1
run_cmd sesdev create ses6 --non-interactive --single-node --qa-test ses6-1node
run_cmd sesdev destroy --non-interactive ses6-1node
run_cmd sesdev create ses6 --non-interactive ses6-4node
Expand All @@ -205,7 +205,7 @@ if [ "$SES6" ] ; then
fi

if [ "$OCTOPUS" ] ; then
run_cmd sesdev box remove --non-interactive leap-15.2 || true
sesdev box remove --non-interactive leap-15.2
run_cmd sesdev create octopus --non-interactive $CEPH_SALT_FROM_SOURCE --single-node --qa-test octopus-1node
run_cmd sesdev destroy --non-interactive octopus-1node
run_cmd sesdev create octopus --non-interactive $CEPH_SALT_FROM_SOURCE octopus-4node
Expand All @@ -214,7 +214,7 @@ if [ "$OCTOPUS" ] ; then
fi

if [ "$SES7" ] ; then
run_cmd sesdev box remove --non-interactive sles-15-sp2 || true
sesdev box remove --non-interactive sles-15-sp2
run_cmd sesdev create ses7 --non-interactive $CEPH_SALT_FROM_SOURCE --single-node --qa-test ses7-1node
run_cmd sesdev destroy --non-interactive ses7-1node
run_cmd sesdev create ses7 --non-interactive $CEPH_SALT_FROM_SOURCE ses7-4node
Expand All @@ -226,7 +226,7 @@ if [ "$SES7" ] ; then
fi

if [ "$PACIFIC" ] ; then
run_cmd sesdev box remove --non-interactive leap-15.2 || true
sesdev box remove --non-interactive leap-15.2
run_cmd sesdev create pacific --non-interactive $CEPH_SALT_FROM_SOURCE --single-node --qa-test pacific-1node
run_cmd sesdev destroy --non-interactive pacific-1node
run_cmd sesdev create pacific --non-interactive $CEPH_SALT_FROM_SOURCE pacific-4node
Expand Down
1 change: 0 additions & 1 deletion sesdev.spec
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ sed -i -e 's/^\s*lv.qemu_use_session = false$//g' seslib/templates/Vagrantfile.j
%install
%py3_install
%fdupes %{buildroot}%{python3_sitelib}
# qa script installation
install -m 0755 -d %{buildroot}/%{_datadir}/%{name}/qa
install -m 0755 -d %{buildroot}/%{_datadir}/%{name}/qa/common
install -m 0755 qa/health-ok.sh %{buildroot}/%{_datadir}/%{name}/qa/health-ok.sh
Expand Down
4 changes: 2 additions & 2 deletions seslib/templates/ceph-salt/ceph_salt_deployment.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ zypper -n in ceph-salt
{% endif %}

systemctl restart salt-master
{% include "wait_for_minions.sh" %}
{% include "wait_for_minions.sh.j2" %}

{% if stop_before_ceph_salt_config %}
exit 0
Expand Down Expand Up @@ -132,4 +132,4 @@ echo "{\"service_type\": \"osd\", \"placement\": {\"host_pattern\": \"{{ node.na
ceph fs volume create myfs "$MDS_NODES_COMMA_SEPARATED_LIST"
{% endif %} {# if ceph_salt_deploy_mdss #}

{% include "qa_test.sh" %}
{% include "qa_test.sh.j2" %}
4 changes: 2 additions & 2 deletions seslib/templates/deepsea/deepsea_deployment.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ EOF

chown -R salt:salt /srv

{% include "wait_for_minions.sh" %}
{% include "wait_for_minions.sh.j2" %}

sleep 2
salt '*' grains.set deepsea True
Expand Down Expand Up @@ -143,4 +143,4 @@ sleep 5

echo "deployment complete!"

{% include "qa_test.sh" %}
{% include "qa_test.sh.j2" %}
2 changes: 1 addition & 1 deletion seslib/templates/deepsea/ses5_pre_stage_0.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ SuSEfirewall2 off
sed -i 's/#worker_threads: 5/worker_threads: 10/g' /etc/salt/master
systemctl restart salt-master
systemctl restart salt-minion
{% include "wait_for_minions.sh" %}
{% include "wait_for_minions.sh.j2" %}

cat > /srv/salt/ceph/updates/nop.sls <<EOF
dummy command:
Expand Down
36 changes: 25 additions & 11 deletions seslib/templates/provision.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ systemctl enable salt-minion
systemctl start salt-minion

{% if version == "ses5" or version == "nautilus" or version == "ses6" %}
{% include "sync_clocks.sh.j2" ignore missing %}
{% include "sync_clocks.sh.j2" %}
{% endif %}

{% endif %} {# not suma #}
Expand All @@ -142,26 +142,40 @@ sleep 5
systemctl restart salt-minion
{% endif %} {# node == master #}

while : ; do
PROVISIONED_NODES=`ls -l /tmp/ready-* 2>/dev/null | wc -l`
echo "waiting for nodes (${PROVISIONED_NODES}/{{ nodes|length }})";
[[ "${PROVISIONED_NODES}" != "{{ nodes|length }}" ]] || break
sleep 2;
set +x
rm -f /tmp/ready-*
echo "Waiting for {{ nodes | length }} nodes to complete initial provisioning..."
while true ; do
{% for node in nodes %}
scp -o StrictHostKeyChecking=no {{ node.name }}:/tmp/ready /tmp/ready-{{ node.name }};
scp -o StrictHostKeyChecking=no {{ node.name }}:/tmp/ready /tmp/ready-{{ node.name }} || true
{% endfor %}
PROVISIONED_NODES="$(find /tmp -maxdepth 1 -type f -regex '/tmp/ready-.*' | wc --lines)"
echo "${PROVISIONED_NODES} of {{ nodes | length }} have completed initial provisioning."
[ "${PROVISIONED_NODES}" = "{{ nodes|length }}" ] && break
set -x
sleep 3
set +x
done
set -x

{% if node == master %}
set +x
echo "Waiting for {{ nodes | length }} minions to submit their keys..."
while true; do
sleep 1
set -x
salt-key -L
N=`salt-key --out=json -L | jq '.minions_pre | length'`
[ "$N" == "{{ nodes|length }}" ] && break
set +x
N=$(salt-key --out=json -L | jq '.minions_pre | length')
echo "${N} of {{ nodes | length }} minions have submitted their keys."
[ "$N" = "{{ nodes|length }}" ] && break
set -x
sleep 3
set +x
done
set -x
salt-key -Ay

{% include "wait_for_minions.sh" %}
{% include "wait_for_minions.sh.j2" %}
{% endif %} {# node == master #}

{% if deployment_tool == "deepsea" %}
Expand Down
File renamed without changes.
21 changes: 0 additions & 21 deletions seslib/templates/wait_for_minions.sh

This file was deleted.

19 changes: 19 additions & 0 deletions seslib/templates/wait_for_minions.sh.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

# make sure all minions are responding
set +ex
LOOP_COUNT="0"
while true ; do
if [ "$LOOP_COUNT" -ge "30" ] ; then
echo "ERROR: minion(s) not responding to ping?"
exit 1
fi
LOOP_COUNT="$((LOOP_COUNT + 1))"
echo "Pinging {{ nodes | length }} minions..."
MINIONS_RESPONDING="$(salt '*' test.ping 2> /dev/null | grep True | wc --lines)"
echo "${MINIONS_RESPONDING} of {{ nodes | length }} minions responded to ping."
[ "$MINIONS_RESPONDING" = "{{ nodes|length }}" ] && break
set -x
sleep 3
set +x
done
set -ex

0 comments on commit 1ef4e80

Please sign in to comment.