vdk-plugins: fix build of multiple plugins (#2445)

There were overall 3 issues that needed to be fixed in the build of
those plugins:

The first one is 2.0 major release of pytest-docker
(https://pypi.org/project/pytest-docker/#history) which we use for
starting docker container in some plugin tests . This impacted us
because

1. pytest-docker change default command from `docker-compose` to `docker
compose` (no dash)
2. But the library we use (docker-compose) installs only
`docker-compose` command (with a dash)

--
The other problematic library is the installation of docker-compose
itself :).
docker-compose depends on PyYaml and PyYAML is having the following
issue: yaml/pyyaml#724 (in short a recent
release of Cython 3.0 broke PyYAML)

As a workaround, we can pin `PyYAML==5.3.1` And also remove eager
(transitive) upgrades when installing the package with pip to reduce the
chance of accidentally upgrading it.

--
Finally removed all non- testing dependencies from requirements.txt. In
one of the plugin was failing to install because it was missing install
requirements in setup.py and was not caught during testing because
requirements.txt was installing that dependency.

------

An alternative approach to fix it might have been to switch to use
[testcontainer](https://pypi.org/project/testcontainers/) (like
vdk-trino uses) instead of pytest-docker + docker-compose. But that
would require refactoring of the tests so I opted out for now.

projects/vdk-plugins/.plugin-common.yml

@@ -9,12 +9,12 @@
 
 .build-plugin:
   stage: build
-  before_script:
+  # before_script: leaving it empty so that script can be extended
+  script:
     - cd projects/vdk-plugins/
     - pip install -U pip
     - pip install ./vdk-test-utils # TODO: REMOVE THIS AT SOME POINT
     - if [ -n "${USE_VDKCORE_DEV_VERSION}" ] ; then pip install -e ../vdk-core; fi
-  script:
     - echo "Build plugin $PLUGIN_NAME"
     - cd ./$PLUGIN_NAME || exit 1
     - ../build-plugin.sh

projects/vdk-plugins/build-plugin.sh

@@ -8,11 +8,10 @@ echo "Building plugin $PLUGIN_NAME"
 
 PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL:-'https://pypi.org/simple'}
 
-pip install -U pip setuptools pre-commit
-pre-commit install --hook-type commit-msg --hook-type pre-commit
+pip install -U pip setuptools
 
 pip install --upgrade --extra-index-url $PIP_EXTRA_INDEX_URL -r requirements.txt
-pip install --upgrade --upgrade-strategy eager -e . --extra-index-url $PIP_EXTRA_INDEX_URL
+pip install --upgrade -e . --extra-index-url $PIP_EXTRA_INDEX_URL
 
 # List exceptions to below check here.
 # Those are not technically plugins so they would not have entry point defined.

projects/vdk-plugins/vdk-greenplum/.plugin-ci.yml

@@ -1,7 +1,7 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-image: "python:3.7"
+
 
 .build-vdk-greenplum:
   variables:

projects/vdk-plugins/vdk-greenplum/requirements.txt

@@ -1,9 +1,8 @@
+docker-compose
 
 # testing requirements
-click
-docker-compose
-psycopg2-binary
-pytest-docker
-tabulate
-vdk-core
+pytest-docker<2
+
+# install earlier version due to https://github.com/yaml/pyyaml/issues/601
+PyYAML==5.3.1
 vdk-test-utils

projects/vdk-plugins/vdk-greenplum/setup.py

@@ -14,7 +14,7 @@
     description="Versatile Data Kit SDK plugin provides support for Greenplum database and greenplum transformation templates.",
     long_description=pathlib.Path("README.md").read_text(),
     long_description_content_type="text/markdown",
-    install_requires=["vdk-core", "psycopg2-binary"],
+    install_requires=["vdk-core", "psycopg2-binary", "tabulate"],
     package_dir={"": "src"},
     packages=setuptools.find_namespace_packages(where="src"),
     include_package_data=True,

projects/vdk-plugins/vdk-impala/.plugin-ci.yml

@@ -1,7 +1,7 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-image: "python:3.7"
+
 
 .build-vdk-impala:
   variables:

projects/vdk-plugins/vdk-impala/requirements.txt

@@ -1,5 +1,8 @@
+docker-compose
 
 # testing requirements
-docker-compose
-pytest-docker
+pytest-docker<2
+
+# install earlier version due to https://github.com/yaml/pyyaml/issues/601
+PyYAML==5.3.1
 vdk-test-utils

projects/vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml

@@ -1,7 +1,7 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-image: "python:3.7"
+
 
 .build-vdk-kerberos-auth:
   variables:

projects/vdk-plugins/vdk-kerberos-auth/requirements.txt

@@ -1,11 +1,11 @@
 docker-compose
+
 # Pinned minikerberos because the current implementation is not compatible with 0.3.0+
 # https://github.com/vmware/versatile-data-kit/issues/1169
-minikerberos==0.2.20
 pytest
-pytest-docker
-requests-kerberos
-vdk-core
+pytest-docker<2
+
+# install earlier version due to https://github.com/yaml/pyyaml/issues/601
+PyYAML==5.3.1
 
-# testing requirements
 vdk-test-utils

projects/vdk-plugins/vdk-postgres/.plugin-ci.yml

@@ -1,7 +1,7 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-image: "python:3.7"
+
 
 .build-vdk-postgres:
   variables:

projects/vdk-plugins/vdk-postgres/requirements.txt

@@ -1,9 +1,8 @@
-
-# testing requirements
-click
 docker-compose
-psycopg2-binary
-pytest-docker
-tabulate
-vdk-core
+# testing requirements
+pytest-docker<2
+
+# install earlier version due to https://github.com/yaml/pyyaml/issues/601
+PyYAML==5.3.1
+
 vdk-test-utils

projects/vdk-plugins/vdk-postgres/setup.py

@@ -14,7 +14,7 @@
     description="Versatile Data Kit SDK plugin provides support for PostgreSQL database and postgres transformation templates.",
     long_description=pathlib.Path("README.md").read_text(),
     long_description_content_type="text/markdown",
-    install_requires=["vdk-core", "psycopg2-binary"],
+    install_requires=["vdk-core", "psycopg2-binary", "tabulate"],
     package_dir={"": "src"},
     packages=setuptools.find_namespace_packages(where="src"),
     include_package_data=True,

projects/vdk-plugins/vdk-trino/requirements.txt

@@ -1,7 +1,7 @@
 
 # testing requirements
 click
-docker-compose
+
 testcontainers
 trino
 vdk-core