From edba589dba3f9a474a0fc293c5b379b6dfdfaea7 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 1 Feb 2025 05:19:08 -0500 Subject: [PATCH] adding milvus md and incorporating feedback from Lokesh Signed-off-by: Francisco Javier Arceo --- docs/reference/online-stores/milvus.md | 64 +++++++++ sdk/python/feast/types.py | 3 + .../online_store/test_online_retrieval.py | 122 +----------------- 3 files changed, 68 insertions(+), 121 deletions(-) create mode 100644 docs/reference/online-stores/milvus.md diff --git a/docs/reference/online-stores/milvus.md b/docs/reference/online-stores/milvus.md new file mode 100644 index 0000000000..9504b7f4de --- /dev/null +++ b/docs/reference/online-stores/milvus.md @@ -0,0 +1,64 @@ +# Redis online store + +## Description + +The [Milvus](https://milvus.io/) online store provides support for materializing feature values into Milvus. + +* The data model used to store feature values in Milvus is described in more detail [here](../../specs/online\_store\_format.md). + +## Getting started +In order to use this online store, you'll need to install the redis extra (along with the dependency needed for the offline store of choice). E.g. + +`pip install 'feast[milvus]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. + +## Examples + +Connecting to a local MilvusDB instance: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: milvus + path: "data/online_store.db" + connection_string: "localhost:6379" + embedding_dim: 128 + index_type: "FLAT" + metric_type: "COSINE" + username: "username" + password: "password" +``` +{% endcode %} + + +The full set of configuration options is available in [RedisOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.redis.RedisOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Milvus online store. + +| | Milvus | +| :-------------------------------------------------------- |:-------| +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/sdk/python/feast/types.py b/sdk/python/feast/types.py index 5caa8f4586..59980d816a 100644 --- a/sdk/python/feast/types.py +++ b/sdk/python/feast/types.py @@ -247,6 +247,9 @@ def from_feast_type( Args: feast_type: The Feast type to be converted. + Returns: + The corresponding ValueType enum. + Raises: ValueError: The conversion could not be performed. """ diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 60354d487d..3f59a749ef 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -446,126 +446,6 @@ def test_get_online_features_milvus() -> None: full_feature_names=False, ) - # TODO: Need to fix these tests to actually run corecctly. - # Create new FeatureStore object with fast cache invalidation - # cache_ttl = 1 - # fs_fast_ttl = FeatureStore( - # config=RepoConfig( - # registry=RegistryConfig( - # path=store.config.registry.path, - # cache_ttl_seconds=cache_ttl, - # ), - # online_store=store.config.online_store, - # project=store.project, - # provider=store.config.provider, - # entity_key_serialization_version=2, - # ), - # repo_path=store.repo_path, - # ) - # - # # Should download the registry and cache it permanently (or until manually refreshed) - # result = fs_fast_ttl.get_online_features( - # features=[ - # "driver_locations:lon", - # "customer_profile:avg_orders_day", - # "customer_profile:name", - # "customer_driver_combined:trips", - # ], - # entity_rows=[{"driver_id": 1, "customer_id": 5}], - # full_feature_names=False, - # ).to_dict() - # assert result["lon"] == ["1.0"] - # assert result["trips"] == [7] - # - # # Rename the registry.db so that it cant be used for refreshes - # os.rename(store.config.registry.path, store.config.registry.path + "_fake") - # - # # Wait for registry to expire - # time.sleep(cache_ttl) - # - # # Will try to reload registry because it has expired (it will fail because we deleted the actual registry file) - # with pytest.raises(FileNotFoundError): - # fs_fast_ttl.get_online_features( - # features=[ - # "driver_locations:lon", - # "customer_profile:avg_orders_day", - # "customer_profile:name", - # "customer_driver_combined:trips", - # ], - # entity_rows=[{"driver_id": 1, "customer_id": 5}], - # full_feature_names=False, - # ).to_dict() - # - # # Restore registry.db so that we can see if it actually reloads registry - # os.rename(store.config.registry.path + "_fake", store.config.registry.path) - # - # # Test if registry is actually reloaded and whether results return - # result = fs_fast_ttl.get_online_features( - # features=[ - # "driver_locations:lon", - # "customer_profile:avg_orders_day", - # "customer_profile:name", - # "customer_driver_combined:trips", - # ], - # entity_rows=[{"driver_id": 1, "customer_id": 5}], - # full_feature_names=False, - # ).to_dict() - # assert result["lon"] == ["1.0"] - # assert result["trips"] == [7] - # - # # Create a registry with infinite cache (for users that want to manually refresh the registry) - # fs_infinite_ttl = FeatureStore( - # config=RepoConfig( - # registry=RegistryConfig( - # path=store.config.registry.path, cache_ttl_seconds=0 - # ), - # online_store=store.config.online_store, - # project=store.project, - # provider=store.config.provider, - # entity_key_serialization_version=2, - # ) - # ) - # - # # Should return results (and fill the registry cache) - # result = fs_infinite_ttl.get_online_features( - # features=[ - # "driver_locations:lon", - # "customer_profile:avg_orders_day", - # "customer_profile:name", - # "customer_driver_combined:trips", - # ], - # entity_rows=[{"driver_id": 1, "customer_id": 5}], - # full_feature_names=False, - # ).to_dict() - # assert result["lon"] == ["1.0"] - # assert result["trips"] == [7] - # - # # Wait a bit so that an arbitrary TTL would take effect - # time.sleep(2) - # - # # Rename the registry.db so that it cant be used for refreshes - # os.rename(store.config.registry.path, store.config.registry.path + "_fake") - # - # # TTL is infinite so this method should use registry cache - # result = fs_infinite_ttl.get_online_features( - # features=[ - # "driver_locations:lon", - # "customer_profile:avg_orders_day", - # "customer_profile:name", - # "customer_driver_combined:trips", - # ], - # entity_rows=[{"driver_id": 1, "customer_id": 5}], - # full_feature_names=False, - # ).to_dict() - # assert result["lon"] == ["1.0"] - # assert result["trips"] == [7] - # - # # Force registry reload (should fail because file is missing) - # with pytest.raises(FileNotFoundError): - # fs_infinite_ttl.refresh_registry() - # - # # Restore registry.db so that teardown works - # os.rename(store.config.registry.path + "_fake", store.config.registry.path) def test_online_to_df(): @@ -915,7 +795,7 @@ def test_local_milvus() -> None: client.drop_collection(collection_name=COLLECTION_NAME) -def test_milvus_lite_get_online_documents() -> None: +def test_milvus_lite_get_online_documents_v2() -> None: """ Test retrieving documents from the online store in local mode. """