WIP: initial implementation of shutdown behavior (#389)

* initial implementation of shutdown behavior

* use on-success to fail on docker wait issues

* using case on shutdown config and fixed some typos

* spaces not tabs

* small fix. SC2045 - Iterating over ls output is fragile. Use globs

runtime/opt/taupage/init.sh

@@ -20,8 +20,8 @@ START_TIME=$(date +"%s")
 cd $(dirname $0)
 
 # general preparation
-for script in $(ls init.d); do
-    ./init.d/$script
+for script in ./init.d/*; do
+    ./$script
     if [ "$?" -ne 0 ]; then
         echo "ERROR: Failed to start $script" >&2
         exit 1
@@ -94,5 +94,27 @@ else
     echo "ERROR: $RUNTIME failed to start with exit code $result ($ELAPSED_SECONDS seconds elapsed)"
 fi
 
-# finished!
-exit $result
+if [ "$config_shutdown" ] && [ "$config_shutdown" != "never" ]; then
+  echo "INFO: waiting for taupage container"
+  EXITCODE=$(docker wait taupageapp)
+
+  echo "INFO: taupage ended with status ${EXITCODE}"
+  case "$config_shutdown" in
+    on-exit)
+      shutdown -h 1 "taupage ended: status ${EXITCODE} calling shutdown in 1";;
+    on-success)
+      [ "$EXITCODE" -eq 0 ] && shutdown -h 1 "taupage ended successfully: status ${EXITCODE} calling shutdown in 1"
+      exit $EXITCODE
+      ;;
+    *)
+      echo "ERROR: shutdown condition unknown"
+      exit 1
+      ;;
+  esac
+
+else
+  echo "INFO: skipping shutdown expecting never"
+
+  # finished!
+  exit $result
+fi

test-batchdata.yaml

@@ -0,0 +1,27 @@
+#taupage-ami-config
+
+application_id: my-http-test-batch-ami
+application_version: "1.0"
+
+runtime: Docker
+source: fatz/docker-simple-job:latest
+
+environment:
+  EXIT: 0
+
+keep_instance_users: true
+
+cloudwatch_logs:
+  /var/log/syslog: my-syslog-loggroup
+  /var/log/application.log: my-application-loggroup
+
+mount_certs: true
+mount_var_log: true
+
+# Start etcd in single node mode
+etcd_discovery_domain: disable
+
+# start local planb-tokeninfo
+local_planb_tokeninfo: true
+
+shutdown: on-success

test.sh

@@ -187,4 +187,75 @@ for testinstance_type in $testinstance_types; do
 	finally true
 done
 
+# get a batch server
+echo "Starting batch server..."
+result=$(aws ec2 run-instances \
+    --image-id $AMI_ID \
+    --count 1 \
+    --instance-type $instance_type \
+    --instance-initiated-shutdown-behavior terminate\
+    --associate-public-ip-address \
+    --key-name $keypair \
+    --security-group-ids $security_group \
+    --subnet-id $subnet \
+    --output json \
+    --region $region \
+    --user-data file://$(pwd)/test-batchdata.yaml)
+
+instanceid=$(echo $result | jq .Instances\[0\].InstanceId | sed 's/"//g')
+echo "Instance: $instanceid"
+
+aws ec2 create-tags --region $region --resources $instanceid --tags "Key=Name,Value=Taupage AMI Batch Test, Key=Version,Value=$TAUPAGE_VERSION"
+
+while [ true ]; do
+    result=$(aws ec2 describe-instances --region $region --instance-id $instanceid --output json)
+    ip=$(echo $result | jq .Reservations\[0\].Instances\[0\].PublicIpAddress | sed 's/"//g')
+
+    [ ! -z "$ip" ] && [ "$ip" != "null" ] && break
+
+    echo "Waiting for public IP..."
+    sleep 5
+done
+
+echo "IP: $ip"
+
+# wait for server
+while [ true ]; do
+    echo "Waiting for server..."
+
+    set +e
+    ssh $ssh_args ubuntu@$ip echo >/dev/null
+    alive=$?
+    set -e
+
+    if [ $alive -eq 0 ]; then
+        break
+    fi
+
+    sleep 2
+done
+
+terminationcounter=300
+
+while [ true ]; do
+	echo "Waiting for self termination"
+
+	result=$(aws ec2 describe-instances --region $region --instance-id $instanceid --output json)
+	state=$(echo $result | jq .Reservations\[0\].Instances\[0\].State.Name | sed 's/"//g')
+
+	if [ "$state" == "terminated" ]; then
+		break
+	fi
+
+	terminationcounter=$((--terminationcounter))
+
+	if [ "$terminationcounter" -le 0 ]; then
+		"Waited 5min for self termination destroying..."
+		exit 1
+	fi
+
+	sleep 2
+
+done
+
 exit 0