From 61f707b92a4c6c808467d55b8954a2ad1c111747 Mon Sep 17 00:00:00 2001 From: tokoko Date: Sat, 23 Mar 2024 06:23:17 +0000 Subject: [PATCH 1/6] run tests as make commands in workflows Signed-off-by: tokoko --- .github/workflows/pr_integration_tests.yml | 2 +- .github/workflows/pr_local_integration_tests.yml | 7 +------ .github/workflows/unit_tests.yml | 13 +------------ Makefile | 12 ++++-------- 4 files changed, 7 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index b335c0f042..5e7287351b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -167,4 +167,4 @@ jobs: SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread + run: make test-python-integration \ No newline at end of file diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 17ff54b1f8..266cdcc9b9 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -61,9 +61,4 @@ jobs: run: make install-python-ci-dependencies - name: Test local integration tests if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak - env: - FEAST_USAGE: "False" - IS_TEST: "True" - FEAST_LOCAL_ONLINE_CONTAINER: "True" - FEAST_IS_LOCAL_TEST: "True" - run: pytest -n 8 --cov=./ --cov-report=xml --color=yes --integration -k "not gcs_registry and not s3_registry and not test_lambda_materialization and not test_snowflake_materialization" sdk/python/tests + run: make test-python-integration-local diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7e2e3b577a..b111d37031 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -23,17 +23,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Install mysql on macOS - if: startsWith(matrix.os, 'macOS') - run: | - brew install mysql - PATH=$PATH:/usr/local/mysql/bin - - name: Work around Homebrew MySQL being broken - # See https://github.com/Homebrew/homebrew-core/issues/130258 for more details. - if: startsWith(matrix.os, 'macOS') - run: | - brew install zlib - ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib - name: Get pip cache dir id: pip-cache run: | @@ -56,7 +45,7 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test Python - run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests + run: make test-python unit-test-ui: diff --git a/Makefile b/Makefile index 0eac7e03a2..af7e68e1e9 100644 --- a/Makefile +++ b/Makefile @@ -64,24 +64,20 @@ benchmark-python-local: FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests test-python: - FEAST_USAGE=False \ - IS_TEST=True \ - python -m pytest -n 8 sdk/python/tests \ + python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: - FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests + python -m pytest -n 8 --integration --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests test-python-integration-local: @(docker info > /dev/null 2>&1 && \ - FEAST_USAGE=False \ - IS_TEST=True \ FEAST_IS_LOCAL_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ - python -m pytest -n 8 --integration \ + python -m pytest -n 8 --color=yes --integration \ -k "not gcs_registry and \ not s3_registry and \ not test_lambda_materialization and \ - not test_snowflake" \ + not test_snowflake_materialization" \ sdk/python/tests \ ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; From f7b50d6591da4be159ba5ebbe0eb02d5d06a185c Mon Sep 17 00:00:00 2001 From: tokoko Date: Sun, 24 Mar 2024 05:47:15 +0000 Subject: [PATCH 2/6] rename test-python make command to test-python-unit Signed-off-by: tokoko --- .github/workflows/unit_tests.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b111d37031..f3f91bb67f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -45,7 +45,7 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test Python - run: make test-python + run: make test-python-unit unit-test-ui: diff --git a/Makefile b/Makefile index af7e68e1e9..890333d4c3 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ format: format-python format-java lint: lint-python lint-java -test: test-python test-java +test: test-python-unit test-java protos: compile-protos-python compile-protos-docs @@ -63,7 +63,7 @@ benchmark-python: benchmark-python-local: FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests -test-python: +test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: From c445415db06aea356f6355a70a5967218736664a Mon Sep 17 00:00:00 2001 From: tokoko Date: Sun, 24 Mar 2024 11:00:02 +0000 Subject: [PATCH 3/6] add env vars to pytest.ini Signed-off-by: tokoko --- Makefile | 21 +------------ sdk/python/pytest.ini | 6 +++- .../requirements/py3.10-ci-requirements.txt | 31 +++++++++++-------- .../requirements/py3.10-requirements.txt | 6 ++-- .../requirements/py3.9-ci-requirements.txt | 31 +++++++++++-------- .../requirements/py3.9-requirements.txt | 6 ++-- sdk/python/tests/conftest.py | 3 -- setup.py | 1 + 8 files changed, 49 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index 890333d4c3..55896b8c11 100644 --- a/Makefile +++ b/Makefile @@ -83,8 +83,6 @@ test-python-integration-local: test-python-integration-container: @(docker info > /dev/null 2>&1 && \ - FEAST_USAGE=False \ - IS_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ python -m pytest -n 8 --integration sdk/python/tests \ ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; @@ -93,7 +91,6 @@ test-python-universal-spark: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \ PYTEST_PLUGINS=feast.infra.offline_stores.contrib.spark_offline_store.tests \ - FEAST_USAGE=False IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_historical_retrieval_fails_on_validation and \ not test_historical_retrieval_with_validation and \ @@ -117,7 +114,6 @@ test-python-universal-trino: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \ PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \ - FEAST_USAGE=False IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_historical_retrieval_fails_on_validation and \ not test_historical_retrieval_with_validation and \ @@ -144,7 +140,6 @@ test-python-universal-mssql: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.mssql_repo_configuration \ PYTEST_PLUGINS=feast.infra.offline_stores.contrib.mssql_offline_store.tests \ - FEAST_USAGE=False IS_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ python -m pytest -n 8 --integration \ -k "not gcs_registry and \ @@ -162,7 +157,6 @@ test-python-universal-athena: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.athena_repo_configuration \ PYTEST_PLUGINS=feast.infra.offline_stores.contrib.athena_offline_store.tests \ - FEAST_USAGE=False IS_TEST=True \ ATHENA_REGION=ap-northeast-2 \ ATHENA_DATA_SOURCE=AwsDataCatalog \ ATHENA_DATABASE=default \ @@ -186,7 +180,6 @@ test-python-universal-athena: test-python-universal-duckdb: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \ - FEAST_USAGE=False IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_nullable_online_store and \ not gcs_registry and \ @@ -200,8 +193,6 @@ test-python-universal-postgres-offline: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \ PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \ - FEAST_USAGE=False \ - IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_historical_retrieval_with_validation and \ not test_historical_features_persisting and \ @@ -222,8 +213,6 @@ test-python-universal-postgres-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.postgres_repo_configuration \ PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \ - FEAST_USAGE=False \ - IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_universal_cli and \ not test_go_feature_server and \ @@ -243,8 +232,6 @@ test-python-universal-postgres-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ - FEAST_USAGE=False \ - IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_universal_cli and \ not test_go_feature_server and \ @@ -264,8 +251,6 @@ test-python-universal-cassandra: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \ - FEAST_USAGE=False \ - IS_TEST=True \ python -m pytest -x --integration \ sdk/python/tests @@ -273,8 +258,6 @@ test-python-universal-hazelcast: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.hazelcast_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.hazelcast \ - FEAST_USAGE=False \ - IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_universal_cli and \ not test_go_feature_server and \ @@ -294,8 +277,6 @@ test-python-universal-cassandra-no-cloud-providers: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \ - FEAST_USAGE=False \ - IS_TEST=True \ python -m pytest -x --integration \ -k "not test_lambda_materialization_consistency and \ not test_apply_entity_integration and \ @@ -310,7 +291,7 @@ test-python-universal-cassandra-no-cloud-providers: sdk/python/tests test-python-universal: - FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests + python -m pytest -n 8 --integration sdk/python/tests format-python: # Sort diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index 07a5e869dc..83317d36c9 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -1,4 +1,8 @@ [pytest] markers = universal_offline_stores: mark a test as using all offline stores. - universal_online_stores: mark a test as using all online stores. \ No newline at end of file + universal_online_stores: mark a test as using all online stores. + +env = + FEAST_USAGE=False + IS_TEST=True \ No newline at end of file diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 8f0ef90d77..4b1e26e1f3 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -61,11 +61,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.65 +boto3==1.34.69 # via # feast (setup.py) # moto -botocore==1.34.65 +botocore==1.34.69 # via # boto3 # moto @@ -82,7 +82,7 @@ cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth -cassandra-driver==3.29.0 +cassandra-driver==3.29.1 # via feast (setup.py) certifi==2024.2.2 # via @@ -199,7 +199,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.17.1 +google-api-core[grpc]==2.18.0 # via # feast (setup.py) # firebase-admin @@ -211,9 +211,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.122.0 +google-api-python-client==2.123.0 # via firebase-admin -google-auth==2.28.2 +google-auth==2.29.0 # via # google-api-core # google-api-python-client @@ -258,7 +258,7 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.11 +great-expectations==0.18.12 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -331,7 +331,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) -importlib-resources==6.3.1 +importlib-resources==6.4.0 # via feast (setup.py) iniconfig==2.0.0 # via pytest @@ -459,7 +459,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.27.0 +msal==1.28.0 # via # azure-identity # msal-extensions @@ -483,7 +483,7 @@ mypy-protobuf==3.3.0 # via feast (setup.py) nbclient==0.10.0 # via nbconvert -nbconvert==7.16.2 +nbconvert==7.16.3 # via jupyter-server nbformat==5.10.3 # via @@ -581,6 +581,7 @@ prompt-toolkit==3.0.43 # via ipython proto-plus==1.23.0 # via + # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -687,6 +688,7 @@ pytest==7.4.4 # feast (setup.py) # pytest-benchmark # pytest-cov + # pytest-env # pytest-lazy-fixture # pytest-mock # pytest-ordering @@ -696,6 +698,8 @@ pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==4.1.0 # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 # via feast (setup.py) pytest-mock==1.10.4 @@ -773,7 +777,7 @@ requests==2.31.0 # snowflake-connector-python # sphinx # trino -requests-oauthlib==1.4.0 +requests-oauthlib==2.0.0 # via kubernetes responses==0.25.0 # via moto @@ -882,6 +886,7 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest + # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -919,7 +924,7 @@ traitlets==5.14.2 # nbformat trino==0.328.0 # via feast (setup.py) -typeguard==4.1.5 +typeguard==4.2.1 # via feast (setup.py) types-protobuf==3.19.22 # via @@ -984,7 +989,7 @@ urllib3==1.26.18 # requests # responses # rockset -uvicorn[standard]==0.28.0 +uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 # via uvicorn diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index e17a588538..6603171d45 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -62,7 +62,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) -importlib-resources==6.3.1 +importlib-resources==6.4.0 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) @@ -158,7 +158,7 @@ toolz==0.12.1 # partd tqdm==4.66.2 # via feast (setup.py) -typeguard==4.1.5 +typeguard==4.2.1 # via feast (setup.py) types-protobuf==4.24.0.20240311 # via mypy-protobuf @@ -176,7 +176,7 @@ tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -uvicorn[standard]==0.28.0 +uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index dc96554431..99dee08c05 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -61,11 +61,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.65 +boto3==1.34.69 # via # feast (setup.py) # moto -botocore==1.34.65 +botocore==1.34.69 # via # boto3 # moto @@ -82,7 +82,7 @@ cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth -cassandra-driver==3.29.0 +cassandra-driver==3.29.1 # via feast (setup.py) certifi==2024.2.2 # via @@ -199,7 +199,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.17.1 +google-api-core[grpc]==2.18.0 # via # feast (setup.py) # firebase-admin @@ -211,9 +211,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.122.0 +google-api-python-client==2.123.0 # via firebase-admin -google-auth==2.28.2 +google-auth==2.29.0 # via # google-api-core # google-api-python-client @@ -258,7 +258,7 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.11 +great-expectations==0.18.12 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -339,7 +339,7 @@ importlib-metadata==6.11.0 # nbconvert # sphinx # typeguard -importlib-resources==6.3.1 +importlib-resources==6.4.0 # via feast (setup.py) iniconfig==2.0.0 # via pytest @@ -467,7 +467,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.27.0 +msal==1.28.0 # via # azure-identity # msal-extensions @@ -491,7 +491,7 @@ mypy-protobuf==3.3.0 # via feast (setup.py) nbclient==0.10.0 # via nbconvert -nbconvert==7.16.2 +nbconvert==7.16.3 # via jupyter-server nbformat==5.10.3 # via @@ -589,6 +589,7 @@ prompt-toolkit==3.0.43 # via ipython proto-plus==1.23.0 # via + # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -695,6 +696,7 @@ pytest==7.4.4 # feast (setup.py) # pytest-benchmark # pytest-cov + # pytest-env # pytest-lazy-fixture # pytest-mock # pytest-ordering @@ -704,6 +706,8 @@ pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==4.1.0 # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 # via feast (setup.py) pytest-mock==1.10.4 @@ -781,7 +785,7 @@ requests==2.31.0 # snowflake-connector-python # sphinx # trino -requests-oauthlib==1.4.0 +requests-oauthlib==2.0.0 # via kubernetes responses==0.25.0 # via moto @@ -892,6 +896,7 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest + # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -929,7 +934,7 @@ traitlets==5.14.2 # nbformat trino==0.328.0 # via feast (setup.py) -typeguard==4.1.5 +typeguard==4.2.1 # via feast (setup.py) types-protobuf==3.19.22 # via @@ -998,7 +1003,7 @@ urllib3==1.26.18 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.28.0 +uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 # via uvicorn diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index f2228ade02..3b8f555ca7 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -63,7 +63,7 @@ importlib-metadata==6.11.0 # dask # feast (setup.py) # typeguard -importlib-resources==6.3.1 +importlib-resources==6.4.0 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) @@ -159,7 +159,7 @@ toolz==0.12.1 # partd tqdm==4.66.2 # via feast (setup.py) -typeguard==4.1.5 +typeguard==4.2.1 # via feast (setup.py) types-protobuf==4.24.0.20240311 # via mypy-protobuf @@ -178,7 +178,7 @@ tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -uvicorn[standard]==0.28.0 +uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 # via uvicorn diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 743a1ce4a0..1c9a958ce3 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -13,7 +13,6 @@ # limitations under the License. import logging import multiprocessing -import os import random from datetime import datetime, timedelta from multiprocessing import Process @@ -24,8 +23,6 @@ import pytest from _pytest.nodes import Item -os.environ["FEAST_USAGE"] = "False" -os.environ["IS_TEST"] = "True" from feast.feature_store import FeatureStore # noqa: E402 from feast.wait import wait_retry_backoff # noqa: E402 from tests.data.data_creator import create_basic_driver_dataset # noqa: E402 diff --git a/setup.py b/setup.py index b32d03ed77..f142da7ec3 100644 --- a/setup.py +++ b/setup.py @@ -177,6 +177,7 @@ "pytest-timeout==1.4.2", "pytest-ordering~=0.6.0", "pytest-mock==1.10.4", + "pytest-env", "Sphinx>4.0.0,<7", "testcontainers>=3.5,<4", "firebase-admin>=5.2.0,<6", From 750aa890d6b7140084e3c115c3c6ed63878872ee Mon Sep 17 00:00:00 2001 From: tokoko Date: Sun, 24 Mar 2024 11:21:36 +0000 Subject: [PATCH 4/6] fix odfv transformation field signature Signed-off-by: tokoko --- sdk/python/feast/on_demand_feature_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 586286a3d4..69c4bb938f 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -82,7 +82,7 @@ def __init__( # noqa: C901 ], udf: Optional[FunctionType] = None, udf_string: str = "", - transformation: Optional[Union[OnDemandPandasTransformation]] = None, + transformation: Optional[Union[OnDemandPandasTransformation, OnDemandSubstraitTransformation]] = None, description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", From 13e7702046fb57caae9b91cb44e1dc72cf474fa8 Mon Sep 17 00:00:00 2001 From: tokoko Date: Sun, 24 Mar 2024 11:35:58 +0000 Subject: [PATCH 5/6] fix linting, formatting Signed-off-by: tokoko --- sdk/python/feast/infra/registry/base_registry.py | 2 +- sdk/python/feast/on_demand_feature_view.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index 9ee3bbbabc..ab841f609b 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -665,7 +665,7 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: odfv_dict["spec"]["userDefinedFunction"][ "body" - ] = on_demand_feature_view.transformation.udf_string + ] = on_demand_feature_view.transformation.udf_string # type: ignore registry_dict["onDemandFeatureViews"].append(odfv_dict) for request_feature_view in sorted( self.list_request_feature_views(project=project), diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 69c4bb938f..3c16046a29 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -62,7 +62,7 @@ class OnDemandFeatureView(BaseFeatureView): features: List[Field] source_feature_view_projections: Dict[str, FeatureViewProjection] source_request_sources: Dict[str, RequestSource] - transformation: Union[OnDemandPandasTransformation] + transformation: Union[OnDemandPandasTransformation, OnDemandSubstraitTransformation] description: str tags: Dict[str, str] owner: str @@ -82,7 +82,9 @@ def __init__( # noqa: C901 ], udf: Optional[FunctionType] = None, udf_string: str = "", - transformation: Optional[Union[OnDemandPandasTransformation, OnDemandSubstraitTransformation]] = None, + transformation: Optional[ + Union[OnDemandPandasTransformation, OnDemandSubstraitTransformation] + ] = None, description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", From d23531fd6ae139e12ea64e30c323a8f91ba4afd8 Mon Sep 17 00:00:00 2001 From: tokoko Date: Tue, 26 Mar 2024 03:44:45 +0000 Subject: [PATCH 6/6] handle stream_feature_view failures Signed-off-by: tokoko --- sdk/python/feast/stream_feature_view.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 0d1125d2bd..301cf6cba5 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -241,11 +241,11 @@ def from_proto(cls, sfv_proto): if sfv_proto.spec.HasField("user_defined_function") else None ) - feature_transformation = ( - sfv_proto.spec.feature_transformation.user_defined_function.body_text - if sfv_proto.spec.HasField("feature_transformation") - else None - ) + # feature_transformation = ( + # sfv_proto.spec.feature_transformation.user_defined_function.body_text + # if sfv_proto.spec.HasField("feature_transformation") + # else None + # ) stream_feature_view = cls( name=sfv_proto.spec.name, description=sfv_proto.spec.description, @@ -264,7 +264,9 @@ def from_proto(cls, sfv_proto): mode=sfv_proto.spec.mode, udf=udf, udf_string=udf_string, - feature_transformation=feature_transformation, + feature_transformation=PandasTransformation(udf, udf_string) + if udf + else None, aggregations=[ Aggregation.from_proto(agg_proto) for agg_proto in sfv_proto.spec.aggregations