From 02e550b7d295ce94a8e9472c22ec88f9850300d1 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Fri, 9 Feb 2018 10:34:59 -0800
Subject: [PATCH 1/3] [TEST] packaging: function to collect debug info

Sometimes when packaging tests fail in CI the test logs aren't enough to
tell what went wrong. This routine helps collect more info about the
state of the es installation at failure time
---
 .../test/resources/packaging/utils/utils.bash | 55 ++++++++++++++++---
 1 file changed, 47 insertions(+), 8 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
index 33d8afb09a041..bd28f2d9b8818 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
@@ -415,6 +415,50 @@ stop_elasticsearch_service() {
     fi
 }
 
+# the default netcat packages in the distributions we test are not all compatible
+test_port() {
+    local host="$1"
+    local port="$2"
+    cat < /dev/null > "/dev/tcp/$host/$port"
+}
+
+describe_port() {
+    local host="$1"
+    local port="$2"
+    run test_port "$host" "$port"
+    if [ "$status" -eq 0 ]; then
+        echo "port $port on host $host is open"
+    else
+        echo "port $port on host $host is not open"
+    fi
+}
+
+debug_collect_logs() {
+    local es_logfile="$ESLOG/elasticsearch.log"
+    local system_logfile='/var/log/messages'
+
+    if [ -e "$es_logfile" ]; then
+        echo "Here's the elasticsearch log:"
+        cat "$es_logfile"
+    else
+        echo "The elasticsearch log doesn't exist at $es_logfile"
+
+	if [ -e "$system_logfile" ]; then
+            echo "Maybe $system_logfile has something:"
+            tail -n20 "$system_logfile"
+	else
+            echo "The logfile at $system_logfile doesn't exist either"
+        fi
+    fi
+
+    echo "Current java processes:"
+    ps aux | grep java || true
+
+    echo "Testing if ES ports are open:"
+    describe_port 127.0.0.1 9200
+    describe_port 127.0.0.1 9201
+}
+
 # Waits for Elasticsearch to reach some status.
 # $1 - expected status - defaults to green
 wait_for_elasticsearch_status() {
@@ -423,14 +467,9 @@ wait_for_elasticsearch_status() {
 
     echo "Making sure elasticsearch is up..."
     wget -O - --retry-connrefused --waitretry=1 --timeout=120 --tries 120 http://localhost:9200/_cluster/health || {
-          echo "Looks like elasticsearch never started. Here is its log:"
-          if [ -e "$ESLOG/elasticsearch.log" ]; then
-              cat "$ESLOG/elasticsearch.log"
-          else
-              echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:"
-              tail -n20 /var/log/messages
-          fi
-          false
+        echo "Looks like elasticsearch never started"
+        debug_collect_logs
+        false
     }
 
     if [ -z "index" ]; then

From d174eaf0514c267af08c13c4a5724ec346b4cb4e Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 12 Feb 2018 13:19:03 -0800
Subject: [PATCH 2/3] [TEST] packaging: fixes from review

---
 qa/vagrant/src/test/resources/packaging/utils/utils.bash | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
index bd28f2d9b8818..af51cdd121a4d 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
@@ -416,6 +416,8 @@ stop_elasticsearch_service() {
 }
 
 # the default netcat packages in the distributions we test are not all compatible
+# so we use /dev/tcp - a feature of bash which makes tcp connections
+# http://tldp.org/LDP/abs/html/devref1.html#DEVTCP
 test_port() {
     local host="$1"
     local port="$2"
@@ -425,8 +427,7 @@ test_port() {
 describe_port() {
     local host="$1"
     local port="$2"
-    run test_port "$host" "$port"
-    if [ "$status" -eq 0 ]; then
+    if test_port "$host" "$port"; then
         echo "port $port on host $host is open"
     else
         echo "port $port on host $host is not open"
@@ -466,7 +467,7 @@ wait_for_elasticsearch_status() {
     local index=$2
 
     echo "Making sure elasticsearch is up..."
-    wget -O - --retry-connrefused --waitretry=1 --timeout=120 --tries 120 http://localhost:9200/_cluster/health || {
+    wget -O - --retry-connrefused --waitretry=1 --timeout=120 --tries=120 http://localhost:9200/_cluster/health || {
         echo "Looks like elasticsearch never started"
         debug_collect_logs
         false

From e92fe4191ebcbaa606e2dc8d66cfebf936c6471b Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 12 Feb 2018 15:59:36 -0800
Subject: [PATCH 3/3] [TEST] packaging: get both es and system logs

---
 .../src/test/resources/packaging/utils/utils.bash    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
index af51cdd121a4d..2cb84528383b3 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
@@ -443,13 +443,13 @@ debug_collect_logs() {
         cat "$es_logfile"
     else
         echo "The elasticsearch log doesn't exist at $es_logfile"
+    fi
 
-	if [ -e "$system_logfile" ]; then
-            echo "Maybe $system_logfile has something:"
-            tail -n20 "$system_logfile"
-	else
-            echo "The logfile at $system_logfile doesn't exist either"
-        fi
+    if [ -e "$system_logfile" ]; then
+        echo "Here's the tail of the log at $system_logfile:"
+        tail -n20 "$system_logfile"
+    else
+        echo "The logfile at $system_logfile doesn't exist"
     fi
 
     echo "Current java processes:"