vdk-trino: change test container library

This change is part of fixing vdk-trino which is failing regularly in
the nightly builds . See
#1559

Until now we used pytest-docker to start service container, this is
switching it to testcontainers-python.

There are multiple advanrages of using it.

* Supposedly Testcontainers is built with performance in mind and uses a
variety of techniques to minimize container startup time and resource
usage like image reuse, container reuse, asynchronous operations.

* It is far more popular lately -
https://hugovk.github.io/top-pypi-packages/  is a site I frequently used
to decide which library is likely to be more stable/used . It is showing
testcontainers is 2000 places ahead of pytest-docker

* Testcontainers provides a wider range of features for managing
containers, such as starting and stopping them, waiting for them to
become available, and exposing their ports to the host system

* Testcontainers-Python allows you to define and configure containers
using Python code instaed of yaml which pytest-docker required.

Testing Done: ran the test suite of vdk-trino a few times and it was
more stable and quick. I have not run actual benchmarks and recorded
times but it seems abotu twice faster possibly. We will verify this as
part of the nightly builds.

Signed-off-by: Antoni Ivanov <[email protected]>

vdk-trino: debugging commit

Signed-off-by: Antoni Ivanov <[email protected]>

projects/vdk-plugins/vdk-trino/.plugin-ci.yml

@@ -4,9 +4,11 @@
 image: "python:3.7"
 
 .build-vdk-trino:
-  image: docker:19.03.8
+  image: docker:23.0.1
   services:
-    - docker:19.03.8-dind
+    - name: docker:23.0.1-dind
+      # explicitly disable tls to avoid docker startup interruption
+      command: ["--tls=false"]
   variables:
     DOCKER_HOST: tcp://localhost:2375
     DOCKER_DRIVER: overlay2

projects/vdk-plugins/vdk-trino/requirements.txt

@@ -2,7 +2,7 @@
 # testing requirements
 click
 docker-compose
-pytest-docker
+testcontainers
 trino
 vdk-core
 vdk-test-utils

projects/vdk-plugins/vdk-trino/src/vdk/plugin/trino/trino_utils.py

@@ -203,7 +203,7 @@ def perform_safe_move_data_to_table_step(
                         "Then try to rerun the data job OR\n"
                         "3. Report the issue to support team.""",
                 )
-        if result:
+        if result is not None:
             log.debug("Target table was successfully created, and we can drop backup")
             self.drop_table(to_db, backup_table_name)
 

projects/vdk-plugins/vdk-trino/tests/conftest.py

@@ -5,31 +5,17 @@
 from unittest import mock
 
 import pytest
-from vdk.plugin.test_utils.util_funcs import CliEntryBasedTestRunner
-from vdk.plugin.trino import trino_plugin
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import wait_for_logs
+
+TRINO_IMAGE = "trinodb/trino:latest"
 
 VDK_TRINO_HOST = "VDK_TRINO_HOST"
 VDK_DB_DEFAULT_TYPE = "VDK_DB_DEFAULT_TYPE"
 VDK_TRINO_PORT = "VDK_TRINO_PORT"
 VDK_TRINO_USE_SSL = "VDK_TRINO_USE_SSL"
 
 
-def is_responsive(runner):
-    try:
-        result = runner.invoke(["trino-query", "--query", "SELECT 1"])
-        if result.exit_code == 0:
-            return True
-    except:
-        return False
-
-
-@pytest.fixture(scope="session")
-def docker_compose_file(pytestconfig):
-    return os.path.join(
-        os.path.dirname(os.path.abspath(__file__)), "docker-compose.yml"
-    )
-
-
 @pytest.fixture(scope="session")
 @mock.patch.dict(
     os.environ,
@@ -40,19 +26,28 @@ def docker_compose_file(pytestconfig):
         VDK_TRINO_USE_SSL: "False",
     },
 )
-def trino_service(docker_ip, docker_services):
+def trino_service(request):
     """Ensure that Trino service is up and responsive."""
     # os.system("echo Check open ports:")
     # os.system("ss -lntu")
-    runner = CliEntryBasedTestRunner(trino_plugin)
-
-    # give the server some time to start before checking if it is ready
-    # before adding this sleep there were intermittent fails of the CI/CD with error:
-    # requests.exceptions.ConnectionError:
-    #   ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
-    # More info: https://stackoverflow.com/questions/383738/104-connection-reset-by-peer-socket-error-or-when-does-closing-a-socket-resu
-    time.sleep(3)
-
-    docker_services.wait_until_responsive(
-        timeout=30.0, pause=0.3, check=lambda: is_responsive(runner)
-    )
+    port = int(os.environ[VDK_TRINO_PORT])
+    container = DockerContainer(TRINO_IMAGE).with_bind_ports(port, port)
+    try:
+        container.start()
+        wait_for_logs(container, "SERVER STARTED", timeout=120)
+        time.sleep(2)
+        print(
+            f"Trino service started on port {container.get_exposed_port(port)} and host {container.get_container_host_ip()}"
+        )
+    except Exception as e:
+        print(f"Failed to start Trino service: {e}")
+        print(f"Container logs: {container.get_logs()}")
+        raise e
+
+    def stop_container():
+        container.stop()
+        print("Trino service stopped")
+
+    request.addfinalizer(stop_container)
+
+    return container