chore: Run tests as make commands in workflows (#4035)

* run tests as make commands in workflows Signed-off-by: tokoko <[email protected]> * rename test-python make command to test-python-unit Signed-off-by: tokoko <[email protected]> * add env vars to pytest.ini Signed-off-by: tokoko <[email protected]> * fix odfv transformation field signature Signed-off-by: tokoko <[email protected]> * fix linting, formatting Signed-off-by: tokoko <[email protected]> * handle stream_feature_view failures Signed-off-by: tokoko <[email protected]> --------- Signed-off-by: tokoko <[email protected]>
feast-dev · Mar 27, 2024 · c364be4 · c364be4
1 parent cf58ebe
commit c364be4
Show file tree

Hide file tree

Showing 12 changed files with 66 additions and 91 deletions.
diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -167,4 +167,4 @@ jobs:
           SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
           SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
           SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
+        run: make test-python-integration
diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml
@@ -61,9 +61,4 @@ jobs:
         run: make install-python-ci-dependencies
       - name: Test local integration tests
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
-        env:
-          FEAST_USAGE: "False"
-          IS_TEST: "True"
-          FEAST_LOCAL_ONLINE_CONTAINER: "True"
-          FEAST_IS_LOCAL_TEST: "True"
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes --integration -k "not gcs_registry and not s3_registry and not test_lambda_materialization and not test_snowflake_materialization" sdk/python/tests
+        run: make test-python-integration-local
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -23,17 +23,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-      - name: Install mysql on macOS
-        if:  startsWith(matrix.os, 'macOS')
-        run: |
-          brew install mysql
-          PATH=$PATH:/usr/local/mysql/bin
-      - name: Work around Homebrew MySQL being broken
-        # See https://github.com/Homebrew/homebrew-core/issues/130258 for more details.
-        if:  startsWith(matrix.os, 'macOS')
-        run: |
-          brew install zlib
-          ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -56,7 +45,7 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests
+        run: make test-python-unit
 
 
   unit-test-ui:

diff --git a/Makefile b/Makefile
@@ -28,7 +28,7 @@ format: format-python format-java
 
 lint: lint-python lint-java
 
-test: test-python test-java
+test: test-python-unit test-java
 
 protos: compile-protos-python compile-protos-docs
 
@@ -63,32 +63,26 @@ benchmark-python:
 benchmark-python-local:
 	FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
 
-test-python:
-	FEAST_USAGE=False \
-	IS_TEST=True \
-	python -m pytest -n 8 sdk/python/tests \
+test-python-unit:
+	python -m pytest -n 8 --color=yes sdk/python/tests
 
 test-python-integration:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	python -m pytest -n 8 --integration --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
 
 test-python-integration-local:
 	@(docker info > /dev/null 2>&1 && \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		FEAST_IS_LOCAL_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
-		python -m pytest -n 8 --integration \
+		python -m pytest -n 8 --color=yes --integration \
 			-k "not gcs_registry and \
  				not s3_registry and \
  				not test_lambda_materialization and \
- 				not test_snowflake" \
+ 				not test_snowflake_materialization" \
 		sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
 
 test-python-integration-container:
 	@(docker info > /dev/null 2>&1 && \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
 		python -m pytest -n 8 --integration sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
@@ -97,7 +91,6 @@ test-python-universal-spark:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.spark_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_historical_retrieval_fails_on_validation and \
 			not test_historical_retrieval_with_validation and \
@@ -121,7 +114,6 @@ test-python-universal-trino:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_historical_retrieval_fails_on_validation and \
 			not test_historical_retrieval_with_validation and \
@@ -148,7 +140,6 @@ test-python-universal-mssql:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.mssql_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.mssql_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
 	FEAST_LOCAL_ONLINE_CONTAINER=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not gcs_registry and \
@@ -166,7 +157,6 @@ test-python-universal-athena:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.athena_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.athena_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
 	ATHENA_REGION=ap-northeast-2 \
 	ATHENA_DATA_SOURCE=AwsDataCatalog \
 	ATHENA_DATABASE=default \
@@ -190,7 +180,6 @@ test-python-universal-athena:
 test-python-universal-duckdb:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \
-	FEAST_USAGE=False IS_TEST=True \
 	python -m pytest -n 8 --integration \
 		-k "not test_nullable_online_store and \
 			not gcs_registry and \
@@ -204,8 +193,6 @@ test-python-universal-postgres-offline:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_historical_retrieval_with_validation and \
 				not test_historical_features_persisting and \
@@ -226,8 +213,6 @@ test-python-universal-postgres-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.postgres_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -247,8 +232,6 @@ test-python-universal-postgres-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -268,17 +251,13 @@ test-python-universal-cassandra:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
-	FEAST_USAGE=False \
-	IS_TEST=True \
 	python -m pytest -x --integration \
 	sdk/python/tests
 
 test-python-universal-hazelcast:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.hazelcast_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.hazelcast \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -298,8 +277,6 @@ test-python-universal-cassandra-no-cloud-providers:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
-	FEAST_USAGE=False \
-	IS_TEST=True \
 	python -m pytest -x --integration \
 	-k "not test_lambda_materialization_consistency   and \
 	  not test_apply_entity_integration               and \
@@ -314,7 +291,7 @@ test-python-universal-cassandra-no-cloud-providers:
 	sdk/python/tests
 
 test-python-universal:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	python -m pytest -n 8 --integration sdk/python/tests
 
 format-python:
 	# Sort

diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py
@@ -241,11 +241,11 @@ def from_proto(cls, sfv_proto):
             if sfv_proto.spec.HasField("user_defined_function")
             else None
         )
-        feature_transformation = (
-            sfv_proto.spec.feature_transformation.user_defined_function.body_text
-            if sfv_proto.spec.HasField("feature_transformation")
-            else None
-        )
+        # feature_transformation = (
+        #     sfv_proto.spec.feature_transformation.user_defined_function.body_text
+        #     if sfv_proto.spec.HasField("feature_transformation")
+        #     else None
+        # )
         stream_feature_view = cls(
             name=sfv_proto.spec.name,
             description=sfv_proto.spec.description,
@@ -264,7 +264,9 @@ def from_proto(cls, sfv_proto):
             mode=sfv_proto.spec.mode,
             udf=udf,
             udf_string=udf_string,
-            feature_transformation=feature_transformation,
+            feature_transformation=PandasTransformation(udf, udf_string)
+            if udf
+            else None,
             aggregations=[
                 Aggregation.from_proto(agg_proto)
                 for agg_proto in sfv_proto.spec.aggregations

diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini
@@ -1,4 +1,8 @@
 [pytest]
 markers =
     universal_offline_stores: mark a test as using all offline stores.
-    universal_online_stores: mark a test as using all online stores.
+    universal_online_stores: mark a test as using all online stores.
+
+env =
+    FEAST_USAGE=False
+    IS_TEST=True
diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt
@@ -61,11 +61,11 @@ black==22.12.0
     # via feast (setup.py)
 bleach==6.1.0
     # via nbconvert
-boto3==1.34.65
+boto3==1.34.69
     # via
     #   feast (setup.py)
     #   moto
-botocore==1.34.65
+botocore==1.34.69
     # via
     #   boto3
     #   moto
@@ -82,7 +82,7 @@ cachecontrol==0.14.0
     # via firebase-admin
 cachetools==5.3.3
     # via google-auth
-cassandra-driver==3.29.0
+cassandra-driver==3.29.1
     # via feast (setup.py)
 certifi==2024.2.2
     # via
@@ -199,7 +199,7 @@ geojson==2.5.0
     # via rockset
 geomet==0.2.1.post1
     # via cassandra-driver
-google-api-core[grpc]==2.17.1
+google-api-core[grpc]==2.18.0
     # via
     #   feast (setup.py)
     #   firebase-admin
@@ -211,9 +211,9 @@ google-api-core[grpc]==2.17.1
     #   google-cloud-datastore
     #   google-cloud-firestore
     #   google-cloud-storage
-google-api-python-client==2.122.0
+google-api-python-client==2.123.0
     # via firebase-admin
-google-auth==2.28.2
+google-auth==2.29.0
     # via
     #   google-api-core
     #   google-api-python-client
@@ -258,7 +258,7 @@ googleapis-common-protos[grpc]==1.63.0
     #   google-api-core
     #   grpc-google-iam-v1
     #   grpcio-status
-great-expectations==0.18.11
+great-expectations==0.18.12
     # via feast (setup.py)
 greenlet==3.0.3
     # via sqlalchemy
@@ -331,7 +331,7 @@ importlib-metadata==6.11.0
     # via
     #   dask
     #   feast (setup.py)
-importlib-resources==6.3.1
+importlib-resources==6.4.0
     # via feast (setup.py)
 iniconfig==2.0.0
     # via pytest
@@ -459,7 +459,7 @@ moreorless==0.4.0
     # via bowler
 moto==4.2.14
     # via feast (setup.py)
-msal==1.27.0
+msal==1.28.0
     # via
     #   azure-identity
     #   msal-extensions
@@ -483,7 +483,7 @@ mypy-protobuf==3.3.0
     # via feast (setup.py)
 nbclient==0.10.0
     # via nbconvert
-nbconvert==7.16.2
+nbconvert==7.16.3
     # via jupyter-server
 nbformat==5.10.3
     # via
@@ -581,6 +581,7 @@ prompt-toolkit==3.0.43
     # via ipython
 proto-plus==1.23.0
     # via
+    #   google-api-core
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-bigtable
@@ -687,6 +688,7 @@ pytest==7.4.4
     #   feast (setup.py)
     #   pytest-benchmark
     #   pytest-cov
+    #   pytest-env
     #   pytest-lazy-fixture
     #   pytest-mock
     #   pytest-ordering
@@ -696,6 +698,8 @@ pytest-benchmark==3.4.1
     # via feast (setup.py)
 pytest-cov==4.1.0
     # via feast (setup.py)
+pytest-env==1.1.3
+    # via feast (setup.py)
 pytest-lazy-fixture==0.6.3
     # via feast (setup.py)
 pytest-mock==1.10.4
@@ -773,7 +777,7 @@ requests==2.31.0
     #   snowflake-connector-python
     #   sphinx
     #   trino
-requests-oauthlib==1.4.0
+requests-oauthlib==2.0.0
     # via kubernetes
 responses==0.25.0
     # via moto
@@ -882,6 +886,7 @@ tomli==2.0.1
     #   pip-tools
     #   pyproject-hooks
     #   pytest
+    #   pytest-env
 tomlkit==0.12.4
     # via snowflake-connector-python
 toolz==0.12.1
@@ -919,7 +924,7 @@ traitlets==5.14.2
     #   nbformat
 trino==0.328.0
     # via feast (setup.py)
-typeguard==4.1.5
+typeguard==4.2.1
     # via feast (setup.py)
 types-protobuf==3.19.22
     # via
@@ -984,7 +989,7 @@ urllib3==1.26.18
     #   requests
     #   responses
     #   rockset
-uvicorn[standard]==0.28.0
+uvicorn[standard]==0.29.0
     # via feast (setup.py)
 uvloop==0.19.0
     # via uvicorn