Merge pull request #4713 from Enmk/test_runner_for_stateless_tests

Fixed test failures when running clickhouse-server on different host

dbms/tests/clickhouse-test

@@ -341,7 +341,7 @@ def main(args):
                                         if result_is_different:
                                             diff = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE).communicate()[0]
                                             diff = unicode(diff, errors='replace', encoding='utf-8')
-                                            cat = Popen(['cat', '-vet'], stdin=PIPE, stdout=PIPE).communicate(input=diff)[0]
+                                            cat = Popen(['cat', '-vet'], stdin=PIPE, stdout=PIPE).communicate(input=diff.encode(encoding='utf-8', errors='replace'))[0]
 
                                             failure = et.Element("failure", attrib = {"message": "result differs with reference"})
                                             report_testcase.append(failure)
@@ -367,12 +367,13 @@ def main(args):
                         print(colored("Break tests execution", "red"))
                         raise e
                     except:
-                        (exc_type, exc_value) = sys.exc_info()[:2]
+                        import traceback
+                        exc_type, exc_value, tb = sys.exc_info()
                         error = et.Element("error", attrib = {"type": exc_type.__name__, "message": str(exc_value)})
                         report_testcase.append(error)
 
                         failures += 1
-                        print("{0} - Test internal error: {1}\n{2}".format(MSG_FAIL, exc_type.__name__, exc_value))
+                        print("{0} - Test internal error: {1}\n{2}\n{3}".format(MSG_FAIL, exc_type.__name__, exc_value, "\n".join(traceback.format_tb(tb, 10))))
                     finally:
                         dump_report(args.output, suite, name, report_testcase)
 

dbms/tests/queries/0_stateless/00354_host_command_line_option.sh

@@ -3,6 +3,6 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --host=localhost --query="SELECT 1";
-$CLICKHOUSE_CLIENT --host localhost --query "SELECT 1";
-$CLICKHOUSE_CLIENT -hlocalhost -q"SELECT 1";
+clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT 1";
+clickhouse_client_removed_host_parameter --host "${CLICKHOUSE_HOST}" --query "SELECT 1";
+clickhouse_client_removed_host_parameter -h"${CLICKHOUSE_HOST}" -q"SELECT 1";

dbms/tests/queries/0_stateless/00368_format_option_collision.sh

@@ -3,4 +3,4 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --host=localhost --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}'
+clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}'

dbms/tests/queries/0_stateless/00634_logging_shard.sh

@@ -9,6 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 cur_name=$(basename "${BASH_SOURCE[0]}")
 server_logs_file="${CLICKHOUSE_TMP}/${cur_name}_server.logs"
+
 server_logs="--server_logs_file=$server_logs_file"
 rm -f "$server_logs_file"
 

dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh

@@ -9,6 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 cur_name=$(basename "${BASH_SOURCE[0]}")
 server_logs_file=${CLICKHOUSE_TMP}/$cur_name"_server.logs"
+
 server_logs="--server_logs_file=$server_logs_file"
 rm -f "$server_logs_file"
 

dbms/tests/queries/0_stateless/00646_url_engine.python

@@ -5,17 +5,31 @@ import sys
 import tempfile
 import threading
 import os, urllib
+import subprocess
 from io import StringIO
 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 
+CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1')
+CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123')
 
-SERVER_ADDRESS = ('127.0.0.1', 51234)
-SERVER_ADDRESS_STR = 'http://' + ':'.join(str(s) for s in SERVER_ADDRESS) + "/"
-CSV_DATA = os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
+#####################################################################################
+# This test starts an HTTP server and serves data to clickhouse url-engine based table.
+# In order for it to work ip+port of http server (given below) should be
+# accessible from clickhouse server.
+#####################################################################################
+
+# IP-address of this host accessible from outside world.
+HTTP_SERVER_HOST = subprocess.check_output(['hostname', '-i']).decode('utf-8').strip()
+HTTP_SERVER_PORT = int(os.environ.get('CLICKHOUSE_TEST_HOST_EXPOSED_PORT', 51234))
 
+# IP address and port of the HTTP server started from this script.
+HTTP_SERVER_ADDRESS = (HTTP_SERVER_HOST, HTTP_SERVER_PORT)
+HTTP_SERVER_URL_STR = 'http://' + ':'.join(str(s) for s in HTTP_SERVER_ADDRESS) + "/"
+CSV_DATA = os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
 
 def get_ch_answer(query):
-    return urllib.urlopen(os.environ.get('CLICKHOUSE_URL', 'http://localhost:' + os.environ.get('CLICKHOUSE_PORT_HTTP', '8123')), data=query).read()
+    url = os.environ.get('CLICKHOUSE_URL', 'http://{host}:{port}'.format(host=CLICKHOUSE_HOST, port=CLICKHOUSE_PORT_HTTP))
+    return urllib.urlopen(url, data=query).read()
 
 def check_answers(query, answer):
     ch_answer = get_ch_answer(query)
@@ -75,7 +89,7 @@ class CSVHTTPServer(BaseHTTPRequestHandler):
         return
 
 def start_server(requests_amount):
-    httpd = HTTPServer(SERVER_ADDRESS, CSVHTTPServer)
+    httpd = HTTPServer(HTTP_SERVER_ADDRESS, CSVHTTPServer)
 
     def real_func():
         for i in xrange(requests_amount):
@@ -96,12 +110,12 @@ def test_select(table_name="", schema="str String,numuint UInt32,numint Int32,do
 
     if table_name:
         get_ch_answer("drop table if exists {}".format(table_name))
-        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, SERVER_ADDRESS_STR))
+        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, HTTP_SERVER_URL_STR))
 
     for i in xrange(len(requests)):
         tbl = table_name
         if not tbl:
-            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema)
+            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema)
         check_answers(requests[i].format(tbl=tbl), answers[i])
 
     if table_name:
@@ -113,19 +127,19 @@ def test_insert(table_name="", schema="str String,numuint UInt32,numint Int32,do
 
     if table_name:
         get_ch_answer("drop table if exists {}".format(table_name))
-        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, SERVER_ADDRESS_STR))
+        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, HTTP_SERVER_URL_STR))
 
     for req in requests_insert:
         tbl = table_name
         if not tbl:
-            tbl = "table function url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema)
+            tbl = "table function url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema)
         get_ch_answer(req.format(tbl=tbl))
 
 
     for i in xrange(len(requests_select)):
         tbl = table_name
         if not tbl:
-            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema)
+            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema)
         check_answers(requests_select[i].format(tbl=tbl), answers[i])
 
     if table_name:

dbms/tests/queries/0_stateless/00837_minmax_index.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+CLICKHOUSE_CLIENT_OPT="--allow_experimental_data_skipping_indices=1"
+
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 

dbms/tests/queries/0_stateless/00838_unique_index.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+CLICKHOUSE_CLIENT_OPT="--allow_experimental_data_skipping_indices=1"
+
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 

dbms/tests/queries/shell_config.sh

@@ -39,3 +39,10 @@ export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"}
 export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time 10"}
 export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."}
 mkdir -p ${CLICKHOUSE_TMP}
+
+function clickhouse_client_removed_host_parameter()
+{
+	# removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex.
+	# bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable.
+	$(echo "$CLICKHOUSE_CLIENT"  | python -c "import sys, re; print re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read())") "$@"
+}

docker/server/local.Dockerfile

@@ -1,18 +1,19 @@
-FROM ubuntu:18.04
-
+# Since right now we can't set volumes to the docker during build, we split building container in stages:
+# 1. build base container
+# 2. run base conatiner with mounted volumes
+# 3. commit container as image
+# 4. build final container atop that image
+# Middle steps are performed by the bash script.
+
+FROM ubuntu:18.04 as clickhouse-server-base
 ARG gosu_ver=1.10
-ARG CLICKHOUSE_PACKAGES_DIR
 
-COPY ${CLICKHOUSE_PACKAGES_DIR}/clickhouse-*.deb /packages/
+VOLUME /packages/
 
-# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically.
+# update to allow installing dependencies of clickhouse automatically
 RUN apt update; \
     DEBIAN_FRONTEND=noninteractive \
-    apt install -y \
-        /packages/clickhouse-common-static_*.deb \
-        /packages/clickhouse-server_*.deb \
-        locales ;\
-    rm -rf /packages
+    apt install -y locales;
 
 ADD https://github.com/tianon/gosu/releases/download/${gosu_ver}/gosu-amd64 /bin/gosu
 
@@ -21,10 +22,18 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically.
+CMD DEBIAN_FRONTEND=noninteractive \
+    apt install -y \
+        /packages/clickhouse-common-static_*.deb \
+        /packages/clickhouse-server_*.deb ;
+
+FROM clickhouse-server-base:postinstall as clickhouse-server
+
 RUN mkdir /docker-entrypoint-initdb.d
 
-COPY server/docker_related_config.xml /etc/clickhouse-server/config.d/
-COPY server/entrypoint.sh /entrypoint.sh
+COPY docker_related_config.xml /etc/clickhouse-server/config.d/
+COPY entrypoint.sh /entrypoint.sh
 
 RUN chmod +x \
     /entrypoint.sh \

docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile

@@ -1,12 +1,15 @@
-FROM ubuntu:18.10
+# Since right now we can't set volumes to the docker during build, we split building container in stages:
+# 1. build base container
+# 2. run base conatiner with mounted volumes
+# 3. commit container as image
+FROM ubuntu:18.10 as clickhouse-test-runner-base
 
-ARG CLICKHOUSE_PACKAGES_DIR
-COPY ${CLICKHOUSE_PACKAGES_DIR}/clickhouse-*.deb /packages/
+# A volume where directory with clickhouse packages to be mounted,
+# for later installing.
+VOLUME /packages
 
-RUN apt-get update ;\
+CMD apt-get update ;\
 	DEBIAN_FRONTEND=noninteractive \
 	apt install -y /packages/clickhouse-common-static_*.deb \
 		/packages/clickhouse-client_*.deb \
-		/packages/clickhouse-test_*.deb \
-		wait-for-it; \
-	rm -rf /packages
+		/packages/clickhouse-test_*.deb

docker/test/test_runner.sh

@@ -1,31 +1,84 @@
 #!/bin/sh
 
-set -e
+set -e -x
 
-# Run tests in docker
-# OR
-# Build containers from deb packages, copying the tests from the source directory 
+trap 'rc=$?; echo EXITED WITH: $rc; exit $rc' EXIT
+
+# CLI option to prevent rebuilding images, just re-run tests with images leftover from previuos time
+readonly NO_REBUILD_FLAG="--no-rebuild"
 
 readonly CLICKHOUSE_DOCKER_DIR="$(realpath ${1})"
-readonly CLICKHOUSE_PACKAGES_DIR="${2}"
+readonly CLICKHOUSE_PACKAGES_ARG="${2}"
 CLICKHOUSE_SERVER_IMAGE="${3}"
 
-# Build test runner image
-docker build \
-	-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
-	-t clickhouse-statelest-test-runner:local \
-	--build-arg CLICKHOUSE_PACKAGES_DIR="${CLICKHOUSE_PACKAGES_DIR}" \
-	"${CLICKHOUSE_DOCKER_DIR}"
+if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+	readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
+fi
+
+
+# In order to allow packages directory to be anywhere, and to reduce amoun of context sent to the docker daemon,
+# all images are built in multiple stages:
+# 1. build base image, install dependencies
+# 2. run image with volume mounted, install what needed from those volumes
+# 3. tag container as image
+# 4. [optional] build another image atop of tagged.
+
+# TODO: optionally mount most recent clickhouse-test and queries directory from local machine
+
+if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+	docker build \
+		-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
+		--target clickhouse-test-runner-base \
+		-t clickhouse-test-runner-base:preinstall \
+		"${CLICKHOUSE_DOCKER_DIR}/test/stateless"
+
+	docker rm -f clickhouse-test-runner-installing-packages || true
+	docker run \
+		-v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+		--name clickhouse-test-runner-installing-packages \
+		clickhouse-test-runner-base:preinstall
+	docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
+	docker rm -f clickhouse-test-runner-installing-packages || true
+fi
+
+# # Create a bind-volume to the clickhouse-test script file
+# docker volume create --driver local --opt type=none --opt device=/home/enmk/proj/ClickHouse_master/dbms/tests/clickhouse-test --opt o=bind clickhouse-test-script-volume
+# docker volume create --driver local --opt type=none --opt device=/home/enmk/proj/ClickHouse_master/dbms/tests/queries --opt o=bind clickhouse-test-queries-dir-volume
 
 # Build server image (optional) from local packages
 if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then
-	CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse_server:local"
+	CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
 
-	docker build \
-		-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
-		-t "${CLICKHOUSE_SERVER_IMAGE}" \
-		--build-arg CLICKHOUSE_PACKAGES_DIR=${CLICKHOUSE_PACKAGES_DIR} \
-		"${CLICKHOUSE_DOCKER_DIR}"
+	if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+		docker build \
+			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+			--target clickhouse-server-base \
+			-t clickhouse-server-base:preinstall \
+			"${CLICKHOUSE_DOCKER_DIR}/server"
+
+		docker rm -f clickhouse_server_base_installing_server || true
+		docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+			--name clickhouse_server_base_installing_server \
+			clickhouse-server-base:preinstall
+		docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
+
+		docker build \
+			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+			--target clickhouse-server \
+			-t "${CLICKHOUSE_SERVER_IMAGE}" \
+			"${CLICKHOUSE_DOCKER_DIR}/server"
+	fi
 fi
 
-CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" run test-runner
+docker rm -f test-runner || true
+docker-compose down
+CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
+	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+	create \
+	--build --force-recreate
+
+CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
+	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+	run \
+	--name test-runner \
+	test-runner

docker/test/test_runner_docker_compose.yaml

@@ -4,27 +4,31 @@ services:
   clickhouse-server:
     image: ${CLICKHOUSE_SERVER_IMAGE}
     expose:
-      - "8123"
-      - "9000"
-      - "9009"
+      - "8123" # HTTP
+      - "9000" # TCP
+      - "9009" # HTTP-interserver
     restart: "no"
 
   test-runner:
-    image: yandex/clickhouse-statelest-test-runner:local
+    image: clickhouse-statelest-test-runner:local
 
     restart: "no"
     depends_on:
       - clickhouse-server
     environment:
       # these are used by clickhouse-test to point clickhouse-client to the right server
       - CLICKHOUSE_HOST=clickhouse-server
-      - CLICKHOUSE_PORT=8123
+      - CLICKHOUSE_PORT=9009
+      - CLICKHOUSE_TEST_HOST_EXPOSED_PORT=51234
+    expose:
+      # port for any test to serve data to clickhouse-server on rare occasion (like URL-engine tables in 00646),
+      # should match value of CLICKHOUSE_TEST_HOST_EXPOSED_PORT above
+      - "51234"
+
+    # NOTE: Dev-mode: mount newest versions of the queries and clickhouse-test script into container.
+    # volumes:
+    #   - /home/enmk/proj/ClickHouse_master/dbms/tests/queries:/usr/share/clickhouse-test/queries:ro
+    #   - /home/enmk/proj/ClickHouse_master/dbms/tests/clickhouse-test:/usr/bin/clickhouse-test:ro
 
-    entrypoint:
-      - wait-for-it
-      - clickhouse-server:8123
-      - -- 
-      - clickhouse-test
-      # - -c
-      # - `which clickhouse-client`
-      - ${CLICKHOUSE_TEST_ARGS}
+    # String-form instead of list-form to allow multiple arguments in "${CLICKHOUSE_TEST_ARGS}"
+    entrypoint: "clickhouse-test ${CLICKHOUSE_TEST_ARGS}"