From 0b5c7dac4e46956125ec0256425682c37ed8e666 Mon Sep 17 00:00:00 2001 From: Antoni Ivanov Date: Mon, 24 Jul 2023 11:01:05 +0300 Subject: [PATCH] vdk-plugins: fix build of greenplugm,postgres,kerberos,impala plugins --- projects/vdk-plugins/.plugin-common.yml | 4 ++-- projects/vdk-plugins/build-plugin.sh | 5 ++--- projects/vdk-plugins/vdk-greenplum/.plugin-ci.yml | 2 +- projects/vdk-plugins/vdk-greenplum/requirements.txt | 11 +++++------ projects/vdk-plugins/vdk-greenplum/setup.py | 2 +- projects/vdk-plugins/vdk-impala/.plugin-ci.yml | 2 +- projects/vdk-plugins/vdk-impala/requirements.txt | 7 +++++-- .../vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml | 2 +- .../vdk-plugins/vdk-kerberos-auth/requirements.txt | 10 +++++----- projects/vdk-plugins/vdk-postgres/.plugin-ci.yml | 2 +- projects/vdk-plugins/vdk-postgres/requirements.txt | 13 ++++++------- projects/vdk-plugins/vdk-postgres/setup.py | 2 +- projects/vdk-plugins/vdk-trino/requirements.txt | 2 +- 13 files changed, 32 insertions(+), 32 deletions(-) diff --git a/projects/vdk-plugins/.plugin-common.yml b/projects/vdk-plugins/.plugin-common.yml index 219d84b35c..fdb2e2cc7f 100644 --- a/projects/vdk-plugins/.plugin-common.yml +++ b/projects/vdk-plugins/.plugin-common.yml @@ -9,12 +9,12 @@ .build-plugin: stage: build - before_script: + # before_script: leaving it empty so that script can be extended + script: - cd projects/vdk-plugins/ - pip install -U pip - pip install ./vdk-test-utils # TODO: REMOVE THIS AT SOME POINT - if [ -n "${USE_VDKCORE_DEV_VERSION}" ] ; then pip install -e ../vdk-core; fi - script: - echo "Build plugin $PLUGIN_NAME" - cd ./$PLUGIN_NAME || exit 1 - ../build-plugin.sh diff --git a/projects/vdk-plugins/build-plugin.sh b/projects/vdk-plugins/build-plugin.sh index 95379c89ec..35b50bee60 100755 --- a/projects/vdk-plugins/build-plugin.sh +++ b/projects/vdk-plugins/build-plugin.sh @@ -8,11 +8,10 @@ echo "Building plugin $PLUGIN_NAME" PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL:-'https://pypi.org/simple'} -pip install -U pip setuptools pre-commit -pre-commit install --hook-type commit-msg --hook-type pre-commit +pip install -U pip setuptools pip install --upgrade --extra-index-url $PIP_EXTRA_INDEX_URL -r requirements.txt -pip install --upgrade --upgrade-strategy eager -e . --extra-index-url $PIP_EXTRA_INDEX_URL +pip install --upgrade -e . --extra-index-url $PIP_EXTRA_INDEX_URL # List exceptions to below check here. # Those are not technically plugins so they would not have entry point defined. diff --git a/projects/vdk-plugins/vdk-greenplum/.plugin-ci.yml b/projects/vdk-plugins/vdk-greenplum/.plugin-ci.yml index d1d1545cf8..c7cfc7688d 100644 --- a/projects/vdk-plugins/vdk-greenplum/.plugin-ci.yml +++ b/projects/vdk-plugins/vdk-greenplum/.plugin-ci.yml @@ -1,7 +1,7 @@ # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -image: "python:3.7" + .build-vdk-greenplum: variables: diff --git a/projects/vdk-plugins/vdk-greenplum/requirements.txt b/projects/vdk-plugins/vdk-greenplum/requirements.txt index cf7a1b16e0..8e9ec99008 100644 --- a/projects/vdk-plugins/vdk-greenplum/requirements.txt +++ b/projects/vdk-plugins/vdk-greenplum/requirements.txt @@ -1,9 +1,8 @@ +docker-compose # testing requirements -click -docker-compose -psycopg2-binary -pytest-docker -tabulate -vdk-core +pytest-docker<2 + +# install earlier version due to https://github.com/yaml/pyyaml/issues/601 +PyYAML==5.3.1 vdk-test-utils diff --git a/projects/vdk-plugins/vdk-greenplum/setup.py b/projects/vdk-plugins/vdk-greenplum/setup.py index 8510968762..62f0c8f6b5 100644 --- a/projects/vdk-plugins/vdk-greenplum/setup.py +++ b/projects/vdk-plugins/vdk-greenplum/setup.py @@ -14,7 +14,7 @@ description="Versatile Data Kit SDK plugin provides support for Greenplum database and greenplum transformation templates.", long_description=pathlib.Path("README.md").read_text(), long_description_content_type="text/markdown", - install_requires=["vdk-core", "psycopg2-binary"], + install_requires=["vdk-core", "psycopg2-binary", "tabulate"], package_dir={"": "src"}, packages=setuptools.find_namespace_packages(where="src"), include_package_data=True, diff --git a/projects/vdk-plugins/vdk-impala/.plugin-ci.yml b/projects/vdk-plugins/vdk-impala/.plugin-ci.yml index 5bc64ecab3..b011c46179 100644 --- a/projects/vdk-plugins/vdk-impala/.plugin-ci.yml +++ b/projects/vdk-plugins/vdk-impala/.plugin-ci.yml @@ -1,7 +1,7 @@ # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -image: "python:3.7" + .build-vdk-impala: variables: diff --git a/projects/vdk-plugins/vdk-impala/requirements.txt b/projects/vdk-plugins/vdk-impala/requirements.txt index 65e7d8fa8c..8e9ec99008 100644 --- a/projects/vdk-plugins/vdk-impala/requirements.txt +++ b/projects/vdk-plugins/vdk-impala/requirements.txt @@ -1,5 +1,8 @@ +docker-compose # testing requirements -docker-compose -pytest-docker +pytest-docker<2 + +# install earlier version due to https://github.com/yaml/pyyaml/issues/601 +PyYAML==5.3.1 vdk-test-utils diff --git a/projects/vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml b/projects/vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml index c6eb6b828f..d1de0d4a8e 100644 --- a/projects/vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml +++ b/projects/vdk-plugins/vdk-kerberos-auth/.plugin-ci.yml @@ -1,7 +1,7 @@ # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -image: "python:3.7" + .build-vdk-kerberos-auth: variables: diff --git a/projects/vdk-plugins/vdk-kerberos-auth/requirements.txt b/projects/vdk-plugins/vdk-kerberos-auth/requirements.txt index b56655cad1..6562ae510c 100644 --- a/projects/vdk-plugins/vdk-kerberos-auth/requirements.txt +++ b/projects/vdk-plugins/vdk-kerberos-auth/requirements.txt @@ -1,11 +1,11 @@ docker-compose + # Pinned minikerberos because the current implementation is not compatible with 0.3.0+ # https://github.com/vmware/versatile-data-kit/issues/1169 -minikerberos==0.2.20 pytest -pytest-docker -requests-kerberos -vdk-core +pytest-docker<2 + +# install earlier version due to https://github.com/yaml/pyyaml/issues/601 +PyYAML==5.3.1 -# testing requirements vdk-test-utils diff --git a/projects/vdk-plugins/vdk-postgres/.plugin-ci.yml b/projects/vdk-plugins/vdk-postgres/.plugin-ci.yml index 36a8cccc0d..3928699ee9 100644 --- a/projects/vdk-plugins/vdk-postgres/.plugin-ci.yml +++ b/projects/vdk-plugins/vdk-postgres/.plugin-ci.yml @@ -1,7 +1,7 @@ # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -image: "python:3.7" + .build-vdk-postgres: variables: diff --git a/projects/vdk-plugins/vdk-postgres/requirements.txt b/projects/vdk-plugins/vdk-postgres/requirements.txt index cf7a1b16e0..9fc1a87fe0 100644 --- a/projects/vdk-plugins/vdk-postgres/requirements.txt +++ b/projects/vdk-plugins/vdk-postgres/requirements.txt @@ -1,9 +1,8 @@ - -# testing requirements -click docker-compose -psycopg2-binary -pytest-docker -tabulate -vdk-core +# testing requirements +pytest-docker<2 + +# install earlier version due to https://github.com/yaml/pyyaml/issues/601 +PyYAML==5.3.1 + vdk-test-utils diff --git a/projects/vdk-plugins/vdk-postgres/setup.py b/projects/vdk-plugins/vdk-postgres/setup.py index 879c19f765..4e1da007f6 100644 --- a/projects/vdk-plugins/vdk-postgres/setup.py +++ b/projects/vdk-plugins/vdk-postgres/setup.py @@ -14,7 +14,7 @@ description="Versatile Data Kit SDK plugin provides support for PostgreSQL database and postgres transformation templates.", long_description=pathlib.Path("README.md").read_text(), long_description_content_type="text/markdown", - install_requires=["vdk-core", "psycopg2-binary"], + install_requires=["vdk-core", "psycopg2-binary", "tabulate"], package_dir={"": "src"}, packages=setuptools.find_namespace_packages(where="src"), include_package_data=True, diff --git a/projects/vdk-plugins/vdk-trino/requirements.txt b/projects/vdk-plugins/vdk-trino/requirements.txt index f12b88a559..a8c314bb5e 100644 --- a/projects/vdk-plugins/vdk-trino/requirements.txt +++ b/projects/vdk-plugins/vdk-trino/requirements.txt @@ -1,7 +1,7 @@ # testing requirements click -docker-compose + testcontainers trino vdk-core