![](\"https://raw.githubusercontent.com/feathr-ai/feathr/main/docs/images/feature_flow.png\")
"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"80223a02-631c-40c8-91b3-a037249ffff9","showTitle":false,"title":""}},"outputs":[],"source":["from datetime import datetime, timedelta\n","import glob\n","import json\n","from math import sqrt\n","import os\n","from pathlib import Path\n","import requests\n","from tempfile import TemporaryDirectory\n","\n","from azure.identity import AzureCliCredential, DefaultAzureCredential \n","from azure.keyvault.secrets import SecretClient\n","import pandas as pd\n","from pyspark.ml import Pipeline\n","from pyspark.ml.evaluation import RegressionEvaluator\n","from pyspark.ml.feature import VectorAssembler\n","from pyspark.ml.regression import GBTRegressor\n","from pyspark.sql import DataFrame, SparkSession\n","import pyspark.sql.functions as F\n","\n","import feathr\n","from feathr import (\n"," FeathrClient,\n"," # Feature data types\n"," BOOLEAN, FLOAT, INT32, ValueType,\n"," # Feature data sources\n"," INPUT_CONTEXT, HdfsSource,\n"," # Feature aggregations\n"," TypedKey, WindowAggTransformation,\n"," # Feature types and anchor\n"," DerivedFeature, Feature, FeatureAnchor,\n"," # Materialization\n"," BackfillTime, MaterializationSettings, RedisSink,\n"," # Offline feature computation\n"," FeatureQuery, ObservationSettings,\n",")\n","from feathr.datasets import nyc_taxi\n","from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n","from feathr.utils.config import generate_config\n","from feathr.utils.job_utils import get_result_df\n","\n","\n","print(f\"\"\"Feathr version: {feathr.__version__}\n","Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ab35fa01-b392-457e-8fde-7e445a3c39b5","showTitle":false,"title":""}},"source":["## 2. Create Shareable Features with Feathr Feature Definition Configs\n","\n","In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n","Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"09f93a9f-7b33-4d91-8f31-ee3b20991696","showTitle":false,"title":""}},"outputs":[],"source":["RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n","PROJECT_NAME = \"feathr_getting_started\"\n","\n","REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n","\n","# Use a databricks cluster\n","SPARK_CLUSTER = \"databricks\"\n","\n","# Databricks file system path\n","DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"41d3648a-9bc9-40dc-90da-bc82b21ef9b3","showTitle":false,"title":""}},"source":["In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec.\n","\n","Note: When submitting jobs, Databricks recommend to use new clusters for greater reliability. If you want to use an existing all-purpose cluster, you may set\n","`existing_cluster_id': ctx.tags().get('clusterId').get()` to the `databricks_config`, replacing `new_cluster` config values."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"331753d6-1850-47b5-ad97-84b7c01d79d1","showTitle":false,"title":""}},"outputs":[],"source":["# Redis credential\n","os.environ['REDIS_PASSWORD'] = REDIS_KEY\n","\n","# Setup databricks env configs\n","ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n","databricks_config = {\n"," 'run_name': \"FEATHR_FILL_IN\",\n"," # To use an existing all-purpose cluster:\n"," # 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n"," # To use a new job cluster:\n"," 'new_cluster': {\n"," 'spark_version': \"11.2.x-scala2.12\",\n"," 'node_type_id': \"Standard_D3_v2\",\n"," 'num_workers':1,\n"," 'spark_conf': {\n"," 'FEATHR_FILL_IN': \"FEATHR_FILL_IN\",\n"," # Exclude conflicting packages if use feathr <= v0.8.0:\n"," 'spark.jars.excludes': \"commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api\",\n"," },\n"," },\n"," 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n"," 'spark_jar_task': {\n"," 'main_class_name': \"FEATHR_FILL_IN\",\n"," 'parameters': [\"FEATHR_FILL_IN\"],\n"," },\n","}\n","os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n","os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n","os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n","os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee","showTitle":false,"title":""}},"source":["### Configurations\n","\n","Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8cd64e3a-376c-48e6-ba41-5197f3591d48","showTitle":false,"title":""}},"outputs":[],"source":["config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n","\n","with open(config_path, 'r') as f: \n"," print(f.read())"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"58d22dc1-7590-494d-94ca-3e2488c31c8e","showTitle":false,"title":""}},"source":["All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3fef7f2f-df19-4f53-90a5-ff7999ed983d","showTitle":false,"title":""}},"source":["### Initialize Feathr Client"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9713a2df-c7b2-4562-88b0-b7acce3cc43a","showTitle":false,"title":""}},"outputs":[],"source":["client = FeathrClient(config_path=config_path)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c3b64bda-d42c-4a64-b976-0fb604cf38c5","showTitle":false,"title":""}},"source":["### View the NYC taxi fare dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c4ccd7b3-298a-4e5a-8eec-b7e309db393e","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n","\n","# Download the data file\n","df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n","df_raw.limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7430c942-64e5-4b70-b823-16ce1d1b3cee","showTitle":false,"title":""}},"source":["### Defining features with Feathr\n","\n","In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n","\n","* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n","* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n","* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n","\n","Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n","\n","There are two types of features -- anchored features and derivated features:\n","\n","* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n","* **Derived features**: Features that are computed on top of other features.\n","\n","#### Define anchored features\n","\n","A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"75b8d2ed-84df-4446-ae07-5f715434f3ea","showTitle":false,"title":""}},"outputs":[],"source":["TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n","TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"93abbcc2-562b-47e4-ad4c-1fedd7cc64df","showTitle":false,"title":""}},"outputs":[],"source":["# We define f_trip_distance and f_trip_time_duration features separately\n","# so that we can reuse them later for the derived features.\n","f_trip_distance = Feature(\n"," name=\"f_trip_distance\",\n"," feature_type=FLOAT,\n"," transform=\"trip_distance\",\n",")\n","f_trip_time_duration = Feature(\n"," name=\"f_trip_time_duration\",\n"," feature_type=FLOAT,\n"," transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",")\n","\n","features = [\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," Feature(\n"," name=\"f_is_long_trip_distance\",\n"," feature_type=BOOLEAN,\n"," transform=\"trip_distance > 30.0\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_week\",\n"," feature_type=INT32,\n"," transform=\"dayofweek(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_month\",\n"," feature_type=INT32,\n"," transform=\"dayofmonth(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_hour_of_day\",\n"," feature_type=INT32,\n"," transform=\"hour(lpep_dropoff_datetime)\",\n"," ),\n","]\n","\n","# After you have defined features, bring them together to build the anchor to the source.\n","feature_anchor = FeatureAnchor(\n"," name=\"feature_anchor\",\n"," source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n"," features=features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"728d2d5f-c11f-4941-bdc5-48507f5749f1","showTitle":false,"title":""}},"source":["We can define the source with a preprocessing python function."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3cc59a0e-a41b-480e-a84e-ca5443d63143","showTitle":false,"title":""}},"outputs":[],"source":["def preprocessing(df: DataFrame) -> DataFrame:\n"," import pyspark.sql.functions as F\n"," df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n"," return df\n","\n","batch_source = HdfsSource(\n"," name=\"nycTaxiBatchSource\",\n"," path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," preprocessing=preprocessing,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"46f863c4-bb81-434a-a448-6b585031a221","showTitle":false,"title":""}},"source":["For the features with aggregation, the supported functions are as follows:\n","\n","| Aggregation Function | Input Type | Description |\n","| --- | --- | --- |\n","|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n","|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n","|LATEST| Any |Returns the latest not-null values from within the defined time window |"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"a373ecbe-a040-4cd3-9d87-0d5f4c5ba553","showTitle":false,"title":""}},"outputs":[],"source":["agg_key = TypedKey(\n"," key_column=\"DOLocationID\",\n"," key_column_type=ValueType.INT32,\n"," description=\"location id in NYC\",\n"," full_name=\"nyc_taxi.location_id\",\n",")\n","\n","agg_window = \"90d\"\n","\n","# Anchored features with aggregations\n","agg_features = [\n"," Feature(\n"," name=\"f_location_avg_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"AVG\",\n"," window=agg_window,\n"," ),\n"," ),\n"," Feature(\n"," name=\"f_location_max_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"MAX\",\n"," window=agg_window,\n"," ),\n"," ),\n","]\n","\n","agg_feature_anchor = FeatureAnchor(\n"," name=\"agg_feature_anchor\",\n"," source=batch_source, # External data source for feature. Typically a data table.\n"," features=agg_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"149f85e2-fa3c-4895-b0c5-de5543ca9b6d","showTitle":false,"title":""}},"source":["#### Define derived features\n","\n","We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"05633bc3-9118-449b-9562-45fc437576c2","showTitle":false,"title":""}},"outputs":[],"source":["derived_features = [\n"," DerivedFeature(\n"," name=\"f_trip_time_distance\",\n"," feature_type=FLOAT,\n"," input_features=[\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," ],\n"," transform=\"f_trip_distance / f_trip_time_duration\",\n"," )\n","]"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ad102c45-586d-468c-85f0-9454401ef10b","showTitle":false,"title":""}},"source":["### Build features\n","\n","Finally, we build the features."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"91bb5ebb-87e4-470b-b8eb-1c89b351740e","showTitle":false,"title":""}},"outputs":[],"source":["client.build_features(\n"," anchor_list=[feature_anchor, agg_feature_anchor],\n"," derived_feature_list=derived_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"632d5f46-f9e2-41a8-aab7-34f75206e2aa","showTitle":false,"title":""}},"source":["## 3. Create Training Data Using Point-in-Time Correct Feature Join\n","\n","After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n","\n","To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n","what features and how these features should be joined to the observation data. \n","\n","To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"02feabc9-2f2f-43e8-898d-b28082798e98","showTitle":false,"title":""}},"outputs":[],"source":["feature_names = [feature.name for feature in features + agg_features + derived_features]\n","feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FORMAT = \"parquet\"\n","offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"67e81466-c736-47ba-b122-e640642c01cf","showTitle":false,"title":""}},"outputs":[],"source":["# Features that we want to request. Can use a subset of features\n","query = FeatureQuery(\n"," feature_list=feature_names,\n"," key=agg_key,\n",")\n","settings = ObservationSettings(\n"," observation_path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")\n","client.get_offline_features(\n"," observation_settings=settings,\n"," feature_query=query,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n"," execution_configurations=SparkExecutionConfiguration({\n"," \"spark.feathr.outputFormat\": DATA_FORMAT,\n"," }),\n"," output_path=offline_features_path,\n",")\n","\n","client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9871af55-25eb-41ee-a58a-fda74b1a174e","showTitle":false,"title":""}},"outputs":[],"source":["# Show feature results\n","df = get_result_df(\n"," spark=spark,\n"," client=client,\n"," data_format=\"parquet\",\n"," res_url=offline_features_path,\n",")\n","df.select(feature_names).limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f","showTitle":false,"title":""}},"source":["## 4. Train and Evaluate a Prediction Model\n","\n","After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n","\n","Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5a226026-1c7b-48db-8f91-88d5c2ddf023","showTitle":false,"title":""}},"source":["### Load Train and Test Data from the Offline Feature Values"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bd2cdc83-0920-46e8-9454-e5e6e7832ce0","showTitle":false,"title":""}},"outputs":[],"source":["# Train / test split\n","train_df, test_df = (\n"," df # Dataframe that we generated from get_offline_features call.\n"," .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n"," .where(F.col(\"f_trip_time_duration\") > 0)\n"," .fillna(0)\n"," .randomSplit([0.8, 0.2])\n",")\n","\n","print(f\"Num train samples: {train_df.count()}\")\n","print(f\"Num test samples: {test_df.count()}\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd","showTitle":false,"title":""}},"source":["### Build a ML Pipeline\n","\n","Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a254361-63e9-45b2-8c19-40549762eacb","showTitle":false,"title":""}},"outputs":[],"source":["# Generate a feature vector column for SparkML\n","vector_assembler = VectorAssembler(\n"," inputCols=[x for x in df.columns if x in feature_names],\n"," outputCol=\"features\",\n",")\n","\n","# Define a model\n","gbt = GBTRegressor(\n"," featuresCol=\"features\",\n"," maxIter=100,\n"," maxDepth=5,\n"," maxBins=16,\n",")\n","\n","# Create a ML pipeline\n","ml_pipeline = Pipeline(stages=[\n"," vector_assembler,\n"," gbt,\n","])"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bef93538-9591-4247-97b6-289d2055b7b1","showTitle":false,"title":""}},"source":["### Train and Evaluate the Model"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"0c3d5f35-11a3-4644-9992-5860169d8302","showTitle":false,"title":""}},"outputs":[],"source":["# Train a model\n","model = ml_pipeline.fit(train_df)\n","\n","# Make predictions\n","predictions = model.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1f9b584c-6228-4a02-a6c3-9b8dd2b78091","showTitle":false,"title":""}},"outputs":[],"source":["# Evaluate\n","evaluator = RegressionEvaluator(\n"," labelCol=\"label\",\n"," predictionCol=\"prediction\",\n",")\n","\n","rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n","mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n","print(f\"RMSE: {rmse}\\nMAE: {mae}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"25c33abd-6e87-437d-a6a1-86435f065a1e","showTitle":false,"title":""}},"outputs":[],"source":["# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n","predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n","\n","predictions_pdf.plot(\n"," x=\"index\",\n"," y=[\"label\", \"prediction\"],\n"," style=['-', ':'],\n"," figsize=(20, 10),\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"664d78cc-4a92-430c-9e05-565ba904558e","showTitle":false,"title":""}},"outputs":[],"source":["predictions_pdf.plot.scatter(\n"," x=\"label\",\n"," y=\"prediction\",\n"," xlim=(0, 100),\n"," ylim=(0, 100),\n"," figsize=(10, 10),\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8a56d165-c813-4ce0-8ae6-9f4d313c463d","showTitle":false,"title":""}},"source":["## 5. Materialize Feature Values for Online Scoring\n","\n","While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n","\n","Note, only the features anchored to offline data source can be materialized."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"751fa72e-8f94-40a1-994e-3e8315b51d37","showTitle":false,"title":""}},"outputs":[],"source":["materialized_feature_names = [feature.name for feature in agg_features]\n","materialized_feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4d4699ed-42e6-408f-903d-2f799284f4b6","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n","\n"," # Get the last date from the dataset\n"," backfill_timestamp = (\n"," df_raw\n"," .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n"," .agg({TIMESTAMP_COL: \"max\"})\n"," .collect()[0][0]\n"," )\n","\n"," # Time range to materialize\n"," backfill_time = BackfillTime(\n"," start=backfill_timestamp,\n"," end=backfill_timestamp,\n"," step=timedelta(days=1),\n"," )\n","\n"," # Destinations:\n"," # For online store,\n"," redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n","\n"," # For offline store,\n"," # adls_sink = HdfsSink(output_path=)\n","\n"," settings = MaterializationSettings(\n"," name=FEATURE_TABLE_NAME + \".job\", # job name\n"," backfill_time=backfill_time,\n"," sinks=[redis_sink], # or adls_sink\n"," feature_names=materialized_feature_names,\n"," )\n","\n"," client.materialize_features(\n"," settings=settings,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n"," )\n","\n"," client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9","showTitle":false,"title":""}},"source":["Now, you can retrieve features for online scoring as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"424bc9eb-a47f-4b46-be69-8218d55e66ad","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," # Note, to get a single key, you may use client.get_online_features instead\n"," materialized_feature_values = client.multi_get_online_features(\n"," feature_table=FEATURE_TABLE_NAME,\n"," keys=[\"239\", \"265\"],\n"," feature_names=materialized_feature_names,\n"," )\n"," materialized_feature_values"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3596dc71-a363-4b6a-a169-215c89978558","showTitle":false,"title":""}},"source":["## Cleanup"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"b5fb292e-bbb6-4dd7-8e79-c62d9533e820","showTitle":false,"title":""}},"outputs":[],"source":["# Remove temporary files\n","dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"databricks_quickstart_nyc_taxi_demo","notebookOrigID":2365994027381987,"widgets":{"REDIS_KEY":{"currentValue":"","nuid":"d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca","widgetInfo":{"defaultValue":"","label":null,"name":"REDIS_KEY","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}},"RESOURCE_PREFIX":{"currentValue":"","nuid":"87a26035-86fc-4dbd-8dd0-dc546c1c63c1","widgetInfo":{"defaultValue":"","label":null,"name":"RESOURCE_PREFIX","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}}}},"kernelspec":{"display_name":"Python 3.10.4 ('feathr')","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.4"},"vscode":{"interpreter":{"hash":"ddb0e38f168d5afaa0b8ab4851ddd8c14364f1d087c15de6ff2ee5a559aec1f2"}}},"nbformat":4,"nbformat_minor":0}
+{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"843d3142-24ca-4bd1-9e31-b55163804fe3","showTitle":false,"title":""}},"outputs":[],"source":["dbutils.widgets.text(\"RESOURCE_PREFIX\", \"\")\n","dbutils.widgets.text(\"REDIS_KEY\", \"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"384e5e16-7213-4186-9d04-09d03b155534","showTitle":false,"title":""}},"source":["# Feathr Feature Store on Databricks Demo Notebook\n","\n","This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).\n","\n","This notebook is specifically written for Databricks and is relying on some of the Databricks packages such as `dbutils`. The intention here is to provide a \"one click run\" example with minimum configuration. For example:\n","- This notebook skips feature registry which requires running Azure Purview. \n","- To make the online feature query work, you will need to configure the Redis endpoint. \n","\n","The full-fledged notebook can be found from [here](https://github.com/feathr-ai/feathr/blob/main/docs/samples/nyc_taxi_demo.ipynb)."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c2ce58c7-9263-469a-bbb7-43364ddb07b8","showTitle":false,"title":""}},"source":["## Prerequisite\n","\n","To use feathr materialization for online scoring with Redis cache, you may deploy a Redis cluster and set `RESOURCE_PREFIX` and `REDIS_KEY` via Databricks widgets. Note that the deployed Redis host address should be `{RESOURCE_PREFIX}redis.redis.cache.windows.net`. More details about how to deploy the Redis cluster can be found [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html#configurure-redis-cluster).\n","\n","To run this notebook, you'll need to install `feathr` pip package. Here, we install notebook-scoped library. For details, please see [Azure Databricks dependency management document](https://learn.microsoft.com/en-us/azure/databricks/libraries/)."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4609d7ad-ad74-40fc-b97e-f440a0fa0737","showTitle":false,"title":""}},"outputs":[],"source":["!pip install feathr"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c81fa80c-bca6-4ae5-84ad-659a036977bd","showTitle":false,"title":""}},"source":["## Notebook Steps\n","\n","This tutorial demonstrates the key capabilities of Feathr, including:\n","\n","1. Install Feathr and necessary dependencies.\n","1. Create shareable features with Feathr feature definition configs.\n","1. Create training data using point-in-time correct feature join\n","1. Train and evaluate a prediction model.\n","1. Materialize feature values for online scoring.\n","\n","The overall data flow is as follows:\n","\n",""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"80223a02-631c-40c8-91b3-a037249ffff9","showTitle":false,"title":""}},"outputs":[],"source":["from datetime import datetime, timedelta\n","import glob\n","import json\n","from math import sqrt\n","import os\n","from pathlib import Path\n","import requests\n","from tempfile import TemporaryDirectory\n","\n","from azure.identity import AzureCliCredential, DefaultAzureCredential \n","from azure.keyvault.secrets import SecretClient\n","import pandas as pd\n","from pyspark.ml import Pipeline\n","from pyspark.ml.evaluation import RegressionEvaluator\n","from pyspark.ml.feature import VectorAssembler\n","from pyspark.ml.regression import GBTRegressor\n","from pyspark.sql import DataFrame, SparkSession\n","import pyspark.sql.functions as F\n","\n","import feathr\n","from feathr import (\n"," FeathrClient,\n"," # Feature data types\n"," BOOLEAN, FLOAT, INT32, ValueType,\n"," # Feature data sources\n"," INPUT_CONTEXT, HdfsSource,\n"," # Feature aggregations\n"," TypedKey, WindowAggTransformation,\n"," # Feature types and anchor\n"," DerivedFeature, Feature, FeatureAnchor,\n"," # Materialization\n"," BackfillTime, MaterializationSettings, RedisSink,\n"," # Offline feature computation\n"," FeatureQuery, ObservationSettings,\n",")\n","from feathr.datasets import nyc_taxi\n","from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n","from feathr.utils.config import generate_config\n","from feathr.utils.job_utils import get_result_df\n","\n","\n","print(f\"\"\"Feathr version: {feathr.__version__}\n","Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ab35fa01-b392-457e-8fde-7e445a3c39b5","showTitle":false,"title":""}},"source":["## 2. Create Shareable Features with Feathr Feature Definition Configs\n","\n","In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n","Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"09f93a9f-7b33-4d91-8f31-ee3b20991696","showTitle":false,"title":""}},"outputs":[],"source":["RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n","PROJECT_NAME = \"feathr_getting_started\"\n","\n","REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n","\n","# Use a databricks cluster\n","SPARK_CLUSTER = \"databricks\"\n","\n","# Databricks file system path\n","DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"41d3648a-9bc9-40dc-90da-bc82b21ef9b3","showTitle":false,"title":""}},"source":["In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec.\n","\n","Note: When submitting jobs, Databricks recommend to use new clusters for greater reliability. If you want to use an existing all-purpose cluster, you may set\n","`existing_cluster_id': ctx.tags().get('clusterId').get()` to the `databricks_config`, replacing `new_cluster` config values."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"331753d6-1850-47b5-ad97-84b7c01d79d1","showTitle":false,"title":""}},"outputs":[],"source":["# Redis credential\n","os.environ['REDIS_PASSWORD'] = REDIS_KEY\n","\n","# Setup databricks env configs\n","ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n","databricks_config = {\n"," 'run_name': \"FEATHR_FILL_IN\",\n"," # To use an existing all-purpose cluster:\n"," # 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n"," # To use a new job cluster:\n"," 'new_cluster': {\n"," 'spark_version': \"11.2.x-scala2.12\",\n"," 'node_type_id': \"Standard_D3_v2\",\n"," 'num_workers':1,\n"," 'spark_conf': {\n"," 'FEATHR_FILL_IN': \"FEATHR_FILL_IN\",\n"," # Exclude conflicting packages if use feathr <= v0.8.0:\n"," 'spark.jars.excludes': \"commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api\",\n"," },\n"," },\n"," 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n"," 'spark_jar_task': {\n"," 'main_class_name': \"FEATHR_FILL_IN\",\n"," 'parameters': [\"FEATHR_FILL_IN\"],\n"," },\n","}\n","os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n","os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n","os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n","os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee","showTitle":false,"title":""}},"source":["### Configurations\n","\n","Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8cd64e3a-376c-48e6-ba41-5197f3591d48","showTitle":false,"title":""}},"outputs":[],"source":["config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n","\n","with open(config_path, 'r') as f: \n"," print(f.read())"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"58d22dc1-7590-494d-94ca-3e2488c31c8e","showTitle":false,"title":""}},"source":["All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3fef7f2f-df19-4f53-90a5-ff7999ed983d","showTitle":false,"title":""}},"source":["### Initialize Feathr Client"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9713a2df-c7b2-4562-88b0-b7acce3cc43a","showTitle":false,"title":""}},"outputs":[],"source":["client = FeathrClient(config_path=config_path)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c3b64bda-d42c-4a64-b976-0fb604cf38c5","showTitle":false,"title":""}},"source":["### View the NYC taxi fare dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c4ccd7b3-298a-4e5a-8eec-b7e309db393e","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n","\n","# Download the data file\n","df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n","df_raw.limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7430c942-64e5-4b70-b823-16ce1d1b3cee","showTitle":false,"title":""}},"source":["### Defining features with Feathr\n","\n","In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n","\n","* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n","* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n","* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n","\n","Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n","\n","There are two types of features -- anchored features and derivated features:\n","\n","* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n","* **Derived features**: Features that are computed on top of other features.\n","\n","#### Define anchored features\n","\n","A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"75b8d2ed-84df-4446-ae07-5f715434f3ea","showTitle":false,"title":""}},"outputs":[],"source":["TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n","TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"93abbcc2-562b-47e4-ad4c-1fedd7cc64df","showTitle":false,"title":""}},"outputs":[],"source":["# We define f_trip_distance and f_trip_time_duration features separately\n","# so that we can reuse them later for the derived features.\n","f_trip_distance = Feature(\n"," name=\"f_trip_distance\",\n"," feature_type=FLOAT,\n"," transform=\"trip_distance\",\n",")\n","f_trip_time_duration = Feature(\n"," name=\"f_trip_time_duration\",\n"," feature_type=FLOAT,\n"," transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",")\n","\n","features = [\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," Feature(\n"," name=\"f_is_long_trip_distance\",\n"," feature_type=BOOLEAN,\n"," transform=\"trip_distance > 30.0\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_week\",\n"," feature_type=INT32,\n"," transform=\"dayofweek(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_month\",\n"," feature_type=INT32,\n"," transform=\"dayofmonth(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_hour_of_day\",\n"," feature_type=INT32,\n"," transform=\"hour(lpep_dropoff_datetime)\",\n"," ),\n","]\n","\n","# After you have defined features, bring them together to build the anchor to the source.\n","feature_anchor = FeatureAnchor(\n"," name=\"feature_anchor\",\n"," source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n"," features=features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"728d2d5f-c11f-4941-bdc5-48507f5749f1","showTitle":false,"title":""}},"source":["We can define the source with a preprocessing python function."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3cc59a0e-a41b-480e-a84e-ca5443d63143","showTitle":false,"title":""}},"outputs":[],"source":["def preprocessing(df: DataFrame) -> DataFrame:\n"," import pyspark.sql.functions as F\n"," df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n"," return df\n","\n","batch_source = HdfsSource(\n"," name=\"nycTaxiBatchSource\",\n"," path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," preprocessing=preprocessing,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"46f863c4-bb81-434a-a448-6b585031a221","showTitle":false,"title":""}},"source":["For the features with aggregation, the supported functions are as follows:\n","\n","| Aggregation Function | Input Type | Description |\n","| --- | --- | --- |\n","|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n","|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n","|LATEST| Any |Returns the latest not-null values from within the defined time window |"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"a373ecbe-a040-4cd3-9d87-0d5f4c5ba553","showTitle":false,"title":""}},"outputs":[],"source":["agg_key = TypedKey(\n"," key_column=\"DOLocationID\",\n"," key_column_type=ValueType.INT32,\n"," description=\"location id in NYC\",\n"," full_name=\"nyc_taxi.location_id\",\n",")\n","\n","agg_window = \"90d\"\n","\n","# Anchored features with aggregations\n","agg_features = [\n"," Feature(\n"," name=\"f_location_avg_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"AVG\",\n"," window=agg_window,\n"," ),\n"," ),\n"," Feature(\n"," name=\"f_location_max_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"MAX\",\n"," window=agg_window,\n"," ),\n"," ),\n","]\n","\n","agg_feature_anchor = FeatureAnchor(\n"," name=\"agg_feature_anchor\",\n"," source=batch_source, # External data source for feature. Typically a data table.\n"," features=agg_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"149f85e2-fa3c-4895-b0c5-de5543ca9b6d","showTitle":false,"title":""}},"source":["#### Define derived features\n","\n","We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"05633bc3-9118-449b-9562-45fc437576c2","showTitle":false,"title":""}},"outputs":[],"source":["derived_features = [\n"," DerivedFeature(\n"," name=\"f_trip_time_distance\",\n"," feature_type=FLOAT,\n"," input_features=[\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," ],\n"," transform=\"f_trip_distance / f_trip_time_duration\",\n"," )\n","]"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ad102c45-586d-468c-85f0-9454401ef10b","showTitle":false,"title":""}},"source":["### Build features\n","\n","Finally, we build the features."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"91bb5ebb-87e4-470b-b8eb-1c89b351740e","showTitle":false,"title":""}},"outputs":[],"source":["client.build_features(\n"," anchor_list=[feature_anchor, agg_feature_anchor],\n"," derived_feature_list=derived_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"632d5f46-f9e2-41a8-aab7-34f75206e2aa","showTitle":false,"title":""}},"source":["## 3. Create Training Data Using Point-in-Time Correct Feature Join\n","\n","After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n","\n","To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n","what features and how these features should be joined to the observation data. \n","\n","To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"02feabc9-2f2f-43e8-898d-b28082798e98","showTitle":false,"title":""}},"outputs":[],"source":["feature_names = [feature.name for feature in features + agg_features + derived_features]\n","feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FORMAT = \"parquet\"\n","offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"67e81466-c736-47ba-b122-e640642c01cf","showTitle":false,"title":""}},"outputs":[],"source":["# Features that we want to request. Can use a subset of features\n","query = FeatureQuery(\n"," feature_list=feature_names,\n"," key=agg_key,\n",")\n","settings = ObservationSettings(\n"," observation_path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")\n","client.get_offline_features(\n"," observation_settings=settings,\n"," feature_query=query,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n"," execution_configurations=SparkExecutionConfiguration({\n"," \"spark.feathr.outputFormat\": DATA_FORMAT,\n"," }),\n"," output_path=offline_features_path,\n",")\n","\n","client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9871af55-25eb-41ee-a58a-fda74b1a174e","showTitle":false,"title":""}},"outputs":[],"source":["# Show feature results\n","df = get_result_df(\n"," spark=spark,\n"," client=client,\n"," data_format=\"parquet\",\n"," res_url=offline_features_path,\n",")\n","df.select(feature_names).limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f","showTitle":false,"title":""}},"source":["## 4. Train and Evaluate a Prediction Model\n","\n","After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n","\n","Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5a226026-1c7b-48db-8f91-88d5c2ddf023","showTitle":false,"title":""}},"source":["### Load Train and Test Data from the Offline Feature Values"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bd2cdc83-0920-46e8-9454-e5e6e7832ce0","showTitle":false,"title":""}},"outputs":[],"source":["# Train / test split\n","train_df, test_df = (\n"," df # Dataframe that we generated from get_offline_features call.\n"," .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n"," .where(F.col(\"f_trip_time_duration\") > 0)\n"," .fillna(0)\n"," .randomSplit([0.8, 0.2])\n",")\n","\n","print(f\"Num train samples: {train_df.count()}\")\n","print(f\"Num test samples: {test_df.count()}\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd","showTitle":false,"title":""}},"source":["### Build a ML Pipeline\n","\n","Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a254361-63e9-45b2-8c19-40549762eacb","showTitle":false,"title":""}},"outputs":[],"source":["# Generate a feature vector column for SparkML\n","vector_assembler = VectorAssembler(\n"," inputCols=[x for x in df.columns if x in feature_names],\n"," outputCol=\"features\",\n",")\n","\n","# Define a model\n","gbt = GBTRegressor(\n"," featuresCol=\"features\",\n"," maxIter=100,\n"," maxDepth=5,\n"," maxBins=16,\n",")\n","\n","# Create a ML pipeline\n","ml_pipeline = Pipeline(stages=[\n"," vector_assembler,\n"," gbt,\n","])"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bef93538-9591-4247-97b6-289d2055b7b1","showTitle":false,"title":""}},"source":["### Train and Evaluate the Model"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"0c3d5f35-11a3-4644-9992-5860169d8302","showTitle":false,"title":""}},"outputs":[],"source":["# Train a model\n","model = ml_pipeline.fit(train_df)\n","\n","# Make predictions\n","predictions = model.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1f9b584c-6228-4a02-a6c3-9b8dd2b78091","showTitle":false,"title":""}},"outputs":[],"source":["# Evaluate\n","evaluator = RegressionEvaluator(\n"," labelCol=\"label\",\n"," predictionCol=\"prediction\",\n",")\n","\n","rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n","mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n","print(f\"RMSE: {rmse}\\nMAE: {mae}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"25c33abd-6e87-437d-a6a1-86435f065a1e","showTitle":false,"title":""}},"outputs":[],"source":["# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n","predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n","\n","predictions_pdf.plot(\n"," x=\"index\",\n"," y=[\"label\", \"prediction\"],\n"," style=['-', ':'],\n"," figsize=(20, 10),\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"664d78cc-4a92-430c-9e05-565ba904558e","showTitle":false,"title":""}},"outputs":[],"source":["predictions_pdf.plot.scatter(\n"," x=\"label\",\n"," y=\"prediction\",\n"," xlim=(0, 100),\n"," ylim=(0, 100),\n"," figsize=(10, 10),\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8a56d165-c813-4ce0-8ae6-9f4d313c463d","showTitle":false,"title":""}},"source":["## 5. Materialize Feature Values for Online Scoring\n","\n","While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n","\n","Note, only the features anchored to offline data source can be materialized."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"751fa72e-8f94-40a1-994e-3e8315b51d37","showTitle":false,"title":""}},"outputs":[],"source":["materialized_feature_names = [feature.name for feature in agg_features]\n","materialized_feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4d4699ed-42e6-408f-903d-2f799284f4b6","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n","\n"," # Get the last date from the dataset\n"," backfill_timestamp = (\n"," df_raw\n"," .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n"," .agg({TIMESTAMP_COL: \"max\"})\n"," .collect()[0][0]\n"," )\n","\n"," # Time range to materialize\n"," backfill_time = BackfillTime(\n"," start=backfill_timestamp,\n"," end=backfill_timestamp,\n"," step=timedelta(days=1),\n"," )\n","\n"," # Destinations:\n"," # For online store,\n"," redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n","\n"," # For offline store,\n"," # adls_sink = HdfsSink(output_path=)\n","\n"," settings = MaterializationSettings(\n"," name=FEATURE_TABLE_NAME + \".job\", # job name\n"," backfill_time=backfill_time,\n"," sinks=[redis_sink], # or adls_sink\n"," feature_names=materialized_feature_names,\n"," )\n","\n"," client.materialize_features(\n"," settings=settings,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n"," )\n","\n"," client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9","showTitle":false,"title":""}},"source":["Now, you can retrieve features for online scoring as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"424bc9eb-a47f-4b46-be69-8218d55e66ad","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," # Note, to get a single key, you may use client.get_online_features instead\n"," materialized_feature_values = client.multi_get_online_features(\n"," feature_table=FEATURE_TABLE_NAME,\n"," keys=[\"239\", \"265\"],\n"," feature_names=materialized_feature_names,\n"," )\n"," materialized_feature_values"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3596dc71-a363-4b6a-a169-215c89978558","showTitle":false,"title":""}},"source":["## Cleanup"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"b5fb292e-bbb6-4dd7-8e79-c62d9533e820","showTitle":false,"title":""}},"outputs":[],"source":["# Remove temporary files\n","dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"databricks_quickstart_nyc_taxi_demo","notebookOrigID":2365994027381987,"widgets":{"REDIS_KEY":{"currentValue":"","nuid":"d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca","widgetInfo":{"defaultValue":"","label":null,"name":"REDIS_KEY","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}},"RESOURCE_PREFIX":{"currentValue":"","nuid":"87a26035-86fc-4dbd-8dd0-dc546c1c63c1","widgetInfo":{"defaultValue":"","label":null,"name":"RESOURCE_PREFIX","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}}}},"kernelspec":{"display_name":"Python 3.10.8 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.8"},"vscode":{"interpreter":{"hash":"b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"}}},"nbformat":4,"nbformat_minor":0}
From 8e401b4f840104d516a51f03c27a1961601eb754 Mon Sep 17 00:00:00 2001
From: Boli Guan
+
+ );
+};
+
+const LineageNodeComponent = forwardRef
+
+
+ {nodeTitle}
+ {active && (
+
+ {nodeSubtitle}
+
- {!feature && (
-
+ {feature.attributes.transformation.transformExpr && (
+
- Transformation
- {feature.attributes.transformation.transformExpr && (
-
+ {feature.attributes.transformation.limit && (
+
- Entity Key
-
+ {feature.attributes.transformation.groupBy && (
+
- Type
-
+ {feature.attributes.transformation.window && (
+
+ {feature.attributes.transformation.defExpr && (
+ Click on feature node to show metadata and metric details
+Click on feature node to show metadata and metric details
+ )} + {feature?.attributes.transformation && ( +Expression: {feature.attributes.transformation.transformExpr}
+ )} + {feature.attributes.transformation.filter && ( +Filter {feature.attributes.transformation.filter}
+ )} + {feature.attributes.transformation.aggFunc && ( +Aggregation: {feature.attributes.transformation.aggFunc}
)} - {feature?.attributes.transformation && ( -- Expression: {feature.attributes.transformation.transformExpr} -
- )} - {feature.attributes.transformation.filter && ( -Filter {feature.attributes.transformation.filter}
- )} - {feature.attributes.transformation.aggFunc && ( -Aggregation: {feature.attributes.transformation.aggFunc}
- )} - {feature.attributes.transformation.limit && ( -Limit: {feature.attributes.transformation.limit}
- )} - {feature.attributes.transformation.groupBy && ( -Group By: {feature.attributes.transformation.groupBy}
- )} - {feature.attributes.transformation.window && ( -Window: {feature.attributes.transformation.window}
- )} - {feature.attributes.transformation.defExpr && ( -Expression: {feature.attributes.transformation.defExpr}
- )} -Limit: {feature.attributes.transformation.limit}
)} - {feature?.attributes.key && feature.attributes.key.length > 0 && ( -Full name: {feature.attributes.key[0].fullName}
-Description: {feature.attributes.key[0].description}
-Key column: {feature.attributes.key[0].keyColumn}
-- Key column alias: {feature.attributes.key[0].keyColumnAlias} -
-Key column type: {feature.attributes.key[0].keyColumnType}
-Group By: {feature.attributes.transformation.groupBy}
)} - {feature?.attributes.type && ( -Dimension Type: {feature.attributes.type.dimensionType}
-Tensor Category: {feature.attributes.type.tensorCategory}
-Type: {feature.attributes.type.type}
-Value Type: {feature.attributes.type.valType}
-Window: {feature.attributes.transformation.window}
)} -Expression: {feature.attributes.transformation.defExpr}
+ )} + + )} + {feature?.attributes.key && feature.attributes.key.length > 0 && ( +Full name: {feature.attributes.key[0].fullName}
+Description: {feature.attributes.key[0].description}
+Key column: {feature.attributes.key[0].keyColumn}
+Key column alias: {feature.attributes.key[0].keyColumnAlias}
+Key column type: {feature.attributes.key[0].keyColumnType}
+Dimension Type: {feature.attributes.type.dimensionType}
+Tensor Category: {feature.attributes.type.tensorCategory}
+Type: {feature.attributes.type.type}
+Value Type: {feature.attributes.type.valType}
+
+
+
+ );
+};
+
+export default SearchBar;
diff --git a/ui/src/pages/dataSource/dataSourceDetails.tsx b/ui/src/pages/dataSource/dataSourceDetails.tsx
index 2548644b2..af82fe0d7 100644
--- a/ui/src/pages/dataSource/dataSourceDetails.tsx
+++ b/ui/src/pages/dataSource/dataSourceDetails.tsx
@@ -1,112 +1,85 @@
import React from "react";
import { LoadingOutlined } from "@ant-design/icons";
import { useNavigate, useParams } from "react-router-dom";
-import { Alert, Button, Card, Col, Row, Spin, Typography } from "antd";
-import { QueryStatus, useQuery } from "react-query";
+import { Alert, Space, Breadcrumb, PageHeader, Spin, Button } from "antd";
+import { Link } from "react-router-dom";
+import { useQuery } from "react-query";
import { AxiosError } from "axios";
-import { fetchDataSource } from "../../api";
-import { DataSource, DataSourceAttributes } from "../../models/model";
-
-const { Title } = Typography;
-
-type DataSourceKeyProps = { dataSource: DataSource };
-const DataSourceKey = ({ dataSource }: DataSourceKeyProps) => {
- const keys = dataSource.attributes;
- return (
- <>
- {keys && (
-
-
- Name: {keys.name}
-Type: {keys.type}
-Path: {keys.path}
-Preprocessing: {keys.preprocessing}
-Event Timestamp Column: {keys.eventTimestampColumn}
-Timestamp Format: {keys.timestampFormat}
-Qualified Name: {keys.qualifiedName}
-Tags: {JSON.stringify(keys.tags)}
-
-
-
-
- {render(status)}
;
+ return (
+
+
+
+ Data Sources
+
+ Data Source Attributes
+
+ }
+ extra={[
+ ,
+ ,
+ ]}
+ >
+
+ {error &&
+
+
+
+ );
};
export default DataSourceDetails;
diff --git a/ui/src/pages/dataSource/dataSources.tsx b/ui/src/pages/dataSource/dataSources.tsx
index 6d84aa0af..c36db0b12 100644
--- a/ui/src/pages/dataSource/dataSources.tsx
+++ b/ui/src/pages/dataSource/dataSources.tsx
@@ -1,20 +1,27 @@
-import { Card, Typography } from "antd";
+import { PageHeader } from "antd";
+import { useState } from "react";
import { useSearchParams } from "react-router-dom";
-import DataSourceList from "../../components/dataSourceList";
-const { Title } = Typography;
+import DataSourceTable from "./components/DataSourceTable";
+import SearchBar from "./components/SearchBar";
const DataSources = () => {
const [searchParams] = useSearchParams();
- const project = (searchParams.get("project") as string) ?? "";
- const keyword = (searchParams.get("keyword") as string) ?? "";
+
+ const [project, setProject] = useState
-
- Data Sources
-
+
+
);
};
diff --git a/ui/src/pages/feature/components/FeatureForm/index.tsx b/ui/src/pages/feature/components/FeatureForm/index.tsx
new file mode 100644
index 000000000..02f33fe8d
--- /dev/null
+++ b/ui/src/pages/feature/components/FeatureForm/index.tsx
@@ -0,0 +1,87 @@
+import React, { forwardRef, useEffect, useState } from "react";
+import { Button, Form, Input, message } from "antd";
+import { useNavigate } from "react-router-dom";
+import { createFeature, updateFeature } from "@/api";
+import { FeatureAttributes, Feature } from "@/models/model";
+
+export interface FeatureFormProps {
+ isNew: boolean;
+ editMode: boolean;
+ feature?: FeatureAttributes;
+}
+
+const FeatureForm = (props: FeatureFormProps, ref: any) => {
+ const navigate = useNavigate();
+
+ const { isNew, editMode, feature } = props;
+
+ const [createLoading, setCreateLoading] = useState+ {transformation.aggFunc && `Window: ${transformation.window}`} + + ); + }, + }, + { + title: "Action", + fixed: "right", + width: 100, + resize: false, + render: (record: Feature) => { + return ( + + ); + }, + }, + ]; + + const { isLoading, data: tableData } = useQuery
+ {data ? (
+ isSource ? (
+
+ Click on source or feature node to show metadata and metric
+ details
+
+ )
+ )}
+
+
+ );
+};
+
+export default NodeDetails;
diff --git a/ui/src/pages/feature/components/SearchBar/index.tsx b/ui/src/pages/feature/components/SearchBar/index.tsx
new file mode 100644
index 000000000..1a32f28b2
--- /dev/null
+++ b/ui/src/pages/feature/components/SearchBar/index.tsx
@@ -0,0 +1,67 @@
+import React, { useRef } from "react";
+import { Form, Input, Button } from "antd";
+import { useNavigate } from "react-router-dom";
+import ProjectsSelect from "@/components/ProjectsSelect";
+
+export interface SearchValue {
+ project?: string;
+ keyword?: string;
+}
+
+export interface SearchBarProps {
+ defaultValues?: SearchValue;
+ onSearch?: (values: SearchValue) => void;
+}
+
+const { Item } = Form;
+
+const SearchBar = (props: SearchBarProps) => {
+ const [form] = Form.useForm();
+
+ const navigate = useNavigate();
+
+ const { defaultValues, onSearch } = props;
+
+ const timeRef = useRef
+
+
+
+ );
+};
+
+export default SearchBar;
diff --git a/ui/src/pages/feature/featureDetails.tsx b/ui/src/pages/feature/featureDetails.tsx
index 549e5e3f7..fdecb7505 100644
--- a/ui/src/pages/feature/featureDetails.tsx
+++ b/ui/src/pages/feature/featureDetails.tsx
@@ -1,218 +1,116 @@
-import React, { useEffect, useState } from "react";
-import { Alert, Button, Card, Col, Row, Space, Spin, Typography } from "antd";
+import React, { useEffect, useRef, useState } from "react";
+import {
+ Alert,
+ Button,
+ PageHeader,
+ Breadcrumb,
+ Space,
+ Card,
+ Spin,
+ Descriptions,
+} from "antd";
import { LoadingOutlined } from "@ant-design/icons";
-import { useNavigate, useParams } from "react-router-dom";
-import { QueryStatus, useQuery } from "react-query";
+import { Link, useNavigate, useParams } from "react-router-dom";
+import { useQuery } from "react-query";
import { AxiosError } from "axios";
-import { fetchFeature } from "../../api";
-import { Feature, InputFeature } from "../../models/model";
-import { FeatureLineage } from "../../models/model";
-import { fetchFeatureLineages } from "../../api";
-import { Elements } from "react-flow-renderer";
-import Graph from "../../components/graph/graph";
-import { getElements } from "../../components/graph/utils";
-
-const { Title } = Typography;
-
-type FeatureKeyProps = { feature: Feature };
-const FeatureKey = ({ feature }: FeatureKeyProps) => {
- const keys = feature.attributes.key;
- return (
- <>
- {keys && keys.length > 0 && (
-
-
- Full Name: {keys[0].fullName}
-Key Column: {keys[0].keyColumn}
-Description: {keys[0].description}
-Key Column Alias: {keys[0].keyColumnAlias}
-Key Column Type: {keys[0].keyColumnType}
-
-
- Dimension Type: {type.dimensionType}
-Tensor Category: {type.tensorCategory}
-Type: {type.type}
-Value Type: {type.valType}
-
- {transformation.transformExpr && (
-
- Expression: {transformation.transformExpr}
- )} - {transformation.filter &&Filter: {transformation.filter}
} - {transformation.aggFunc && ( -Aggregation: {transformation.aggFunc}
- )} - {transformation.limit &&Limit: {transformation.limit}
} - {transformation.groupBy && ( -Group By: {transformation.groupBy}
- )} - {transformation.window &&Window: {transformation.window}
} - {transformation.defExpr && ( -Expression: {transformation.defExpr}
- )} -
-
-
-
-
-
-
-
-
-
+
+
+ Features
+
+ Feature Details
+
}
- }
- };
-
- return
+ {error &&
+
+
+
+ );
};
export default FeatureDetails;
diff --git a/ui/src/pages/feature/features.tsx b/ui/src/pages/feature/features.tsx
index 275cde11f..9ace6ead6 100644
--- a/ui/src/pages/feature/features.tsx
+++ b/ui/src/pages/feature/features.tsx
@@ -1,20 +1,27 @@
-import { Button, Card, Space, Typography } from "antd";
-import { useNavigate, useSearchParams } from "react-router-dom";
-import FeatureList from "../../components/featureList";
-
-const { Title } = Typography;
+import { useState } from "react";
+import { PageHeader } from "antd";
+import { useSearchParams } from "react-router-dom";
+import SearchBar, { SearchValue } from "./components/SearchBar";
+import FeatureTable from "./components/FeatureTable";
const Features = () => {
const [searchParams] = useSearchParams();
- const project = (searchParams.get("project") as string) ?? "";
- const keyword = (searchParams.get("keyword") as string) ?? "";
+
+ const [search, setProject] = useState{render(status)}
;
+ extra={[
+ ,
+ ]}
+ >
+
-
- Features
-
+
+
);
};
diff --git a/ui/src/pages/feature/lineageGraph.tsx b/ui/src/pages/feature/lineageGraph.tsx
index ac75dff91..d8b1473df 100644
--- a/ui/src/pages/feature/lineageGraph.tsx
+++ b/ui/src/pages/feature/lineageGraph.tsx
@@ -1,17 +1,17 @@
-import React, { useEffect, useState } from "react";
-import { Card, Col, Radio, Row, Spin, Tabs, Typography } from "antd";
+import React, { useEffect, useRef, useState } from "react";
+import { PageHeader, Row, Col, Radio, Tabs } from "antd";
import { useParams, useSearchParams } from "react-router-dom";
-import { Elements } from "react-flow-renderer";
-import Graph from "../../components/graph/graph";
-import { fetchProjectLineages } from "../../api";
-import { FeatureLineage } from "../../models/model";
-import { LoadingOutlined } from "@ant-design/icons";
-import GraphNodeDetails from "../../components/graph/graphNodeDetails";
-import { getElements } from "../../components/graph/utils";
-import { FeatureType } from "../../utils/utils";
+import FlowGraph from "@/components/FlowGraph";
+import { fetchProjectLineages } from "@/api";
+import { FeatureLineage } from "@/models/model";
+import { FeatureType } from "@/utils/utils";
+import NodeDetails from "./components/NodeDetails";
-const { Title } = Typography;
-const { TabPane } = Tabs;
+const items = [
+ { label: "Metadata", key: "1", children: Under construction
}, // 务必填写 key + { label: "Jobs", key: "3", children:Under construction
}, +]; type Params = { project: string; @@ -22,90 +22,75 @@ const LineageGraph = () => { const nodeId = searchParams.get("nodeId") as string; const [lineageData, setLineageData] = useState
-
- Lineage {project}
-
+
+
);
};
diff --git a/ui/src/pages/feature/newFeature.tsx b/ui/src/pages/feature/newFeature.tsx
index d51dd2aa0..50afd64c3 100644
--- a/ui/src/pages/feature/newFeature.tsx
+++ b/ui/src/pages/feature/newFeature.tsx
@@ -1,16 +1,13 @@
import React from "react";
-import { Card, Typography } from "antd";
-import FeatureForm from "../../components/featureForm";
-
-const { Title } = Typography;
+import { PageHeader } from "antd";
+import FeatureForm from "./components/FeatureForm";
const NewFeature = () => {
return (
- toggleFeatureType(e.target.value)}
- >
- All Nodes
- Source
- Anchor
-
- Anchor Feature
-
-
- Derived Feature
-
-
-
-
- {loading ? (
-
+ toggleFeatureType(e.target.value)}
+ >
+ All Nodes
+ Source
+
+ Anchor Feature
+
+
+ Derived Feature
+
+
+
+
-
-
-
-
-
-
-
-
-
-
- )}
-
- Under construction
-Under construction
-
-
- Create Feature
+
+
);
};
diff --git a/ui/src/pages/management/components/RoleForm/index.tsx b/ui/src/pages/management/components/RoleForm/index.tsx
index 9e073abd8..0a77b1610 100644
--- a/ui/src/pages/management/components/RoleForm/index.tsx
+++ b/ui/src/pages/management/components/RoleForm/index.tsx
@@ -1,6 +1,6 @@
import React, { forwardRef, useCallback, useEffect, useState } from "react";
import { Form, Select, Input, Button, message } from "antd";
-import { listUserRole, addUserRole } from "../../../../api";
+import { listUserRole, addUserRole } from "@/api";
export interface RoleFormProps {
getRole?: (isAdmin: boolean) => void;
diff --git a/ui/src/pages/management/components/UserRolesTable/index.tsx b/ui/src/pages/management/components/UserRolesTable/index.tsx
index d264b2691..9a72f1539 100644
--- a/ui/src/pages/management/components/UserRolesTable/index.tsx
+++ b/ui/src/pages/management/components/UserRolesTable/index.tsx
@@ -6,12 +6,12 @@ import React, {
useRef,
useState,
} from "react";
-import { Table, Tag, Button, message, Popconfirm } from "antd";
+import { Tag, Button, message, Popconfirm } from "antd";
import { DeleteOutlined } from "@ant-design/icons";
-import { ColumnsType } from "antd/lib/table";
import dayjs from "dayjs";
-import { UserRole } from "../../../../models/model";
-import { listUserRole, deleteUserRole } from "../../../../api";
+import { UserRole } from "@/models/model";
+import { listUserRole, deleteUserRole } from "@/api";
+import ResizeTable, { ResizeColumnType } from "@/components/ResizeTable";
export interface UserRolesTableProps {}
@@ -74,25 +74,32 @@ const UserRolesTable = (props: UserRolesTableProps, ref: any) => {
}
};
- const columns: ColumnsType
+
+
+ );
+};
+
+const SearchBarComponent = forwardRef
-
- Projects
-
+
+
);
};
diff --git a/ui/src/site.css b/ui/src/site.css
index cea4439f7..e1a42a944 100644
--- a/ui/src/site.css
+++ b/ui/src/site.css
@@ -4,8 +4,6 @@
}
.card {
- margin-top: 15px;
- margin-right: 15px;
box-shadow: 5px 8px 15px 5px rgba(208, 216, 243, 0.6);
border-radius: 8px;
}
@@ -61,3 +59,7 @@
.dataSource-container {
column-count: 1;
}
+
+.display-flex {
+ display: "flex";
+}
diff --git a/ui/src/utils/attributesMapping.ts b/ui/src/utils/attributesMapping.ts
new file mode 100644
index 000000000..09e7459b7
--- /dev/null
+++ b/ui/src/utils/attributesMapping.ts
@@ -0,0 +1,48 @@
+import {
+ FeatureTransformation,
+ FeatureKey,
+ FeatureType,
+ DataSourceAttributes,
+} from "@/models/model";
+
+export const TransformationMap: Array<{
+ label: string;
+ key: keyof FeatureTransformation;
+}> = [
+ { label: "Expression", key: "transformExpr" },
+ { label: "Filter", key: "filter" },
+ { label: "Aggregation", key: "aggFunc" },
+ { label: "Limit", key: "limit" },
+ { label: "Group By", key: "groupBy" },
+ { label: "Window", key: "window" },
+ { label: "Expression", key: "defExpr" },
+];
+
+export const FeatureKeyMap: Array<{ label: string; key: keyof FeatureKey }> = [
+ { label: "Full name", key: "fullName" },
+ { label: "Description", key: "description" },
+ { label: "Key column", key: "keyColumn" },
+ { label: "Key column alias", key: "keyColumnAlias" },
+ { label: "Key column type", key: "keyColumnType" },
+];
+
+export const TypeMap: Array<{ label: string; key: keyof FeatureType }> = [
+ { label: "Dimension Type", key: "dimensionType" },
+ { label: "Tensor Category", key: "tensorCategory" },
+ { label: "Type", key: "type" },
+ { label: "Value Type", key: "valType" },
+];
+
+export const SourceAttributesMap: Array<{
+ label: string;
+ key: keyof DataSourceAttributes;
+}> = [
+ { label: "Name", key: "name" },
+ { label: "Type", key: "type" },
+ { label: "Path", key: "path" },
+ { label: "Preprocessing", key: "preprocessing" },
+ { label: "Event Timestamp Column", key: "event_timestamp_column" },
+ { label: "Timestamp Forma", key: "timestamp_format" },
+ { label: "Qualified Name", key: "qualified_name" },
+ { label: "Tags", key: "tags" },
+];
diff --git a/ui/src/utils/utils.tsx b/ui/src/utils/utils.tsx
index 85bfd8f42..9cd2c959b 100644
--- a/ui/src/utils/utils.tsx
+++ b/ui/src/utils/utils.tsx
@@ -1,3 +1,4 @@
+import { Feature } from "@/models/model";
import { Configuration, PublicClientApplication } from "@azure/msal-browser";
export const getMsalConfig = () => {
@@ -16,8 +17,6 @@ export const getMsalConfig = () => {
redirectUri: window.location.origin,
},
};
- console.log("clientId = ", clientId);
- console.log("authority = ", authority);
return new PublicClientApplication(msalConfig);
};
@@ -33,6 +32,19 @@ export const enum FeatureType {
export const isFeature = (featureType: string) => {
return (
featureType === FeatureType.AnchorFeature ||
- featureType === FeatureType.DerivedFeature
+ featureType === FeatureType.DerivedFeature ||
+ featureType === FeatureType.Source
);
};
+
+export const getFeatureDetailUrl = (project: string, feature: Feature) => {
+ switch (feature.typeName) {
+ case FeatureType.Source:
+ return `/projects/${project}/dataSources/${feature.guid}`;
+ case FeatureType.AnchorFeature:
+ case FeatureType.DerivedFeature:
+ return `/projects/${project}/features/${feature.guid}`;
+ default:
+ return;
+ }
+};
From afd930903be8368c4f0e71596e7ca5bd6ccb3c73 Mon Sep 17 00:00:00 2001
From: Blair Chen "]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"80223a02-631c-40c8-91b3-a037249ffff9","showTitle":false,"title":""}},"outputs":[],"source":["from datetime import datetime, timedelta\n","import glob\n","import json\n","from math import sqrt\n","import os\n","from pathlib import Path\n","import requests\n","from tempfile import TemporaryDirectory\n","\n","from azure.identity import AzureCliCredential, DefaultAzureCredential \n","from azure.keyvault.secrets import SecretClient\n","import pandas as pd\n","from pyspark.ml import Pipeline\n","from pyspark.ml.evaluation import RegressionEvaluator\n","from pyspark.ml.feature import VectorAssembler\n","from pyspark.ml.regression import GBTRegressor\n","from pyspark.sql import DataFrame, SparkSession\n","import pyspark.sql.functions as F\n","\n","import feathr\n","from feathr import (\n"," FeathrClient,\n"," # Feature data types\n"," BOOLEAN, FLOAT, INT32, ValueType,\n"," # Feature data sources\n"," INPUT_CONTEXT, HdfsSource,\n"," # Feature aggregations\n"," TypedKey, WindowAggTransformation,\n"," # Feature types and anchor\n"," DerivedFeature, Feature, FeatureAnchor,\n"," # Materialization\n"," BackfillTime, MaterializationSettings, RedisSink,\n"," # Offline feature computation\n"," FeatureQuery, ObservationSettings,\n",")\n","from feathr.datasets import nyc_taxi\n","from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n","from feathr.utils.config import generate_config\n","from feathr.utils.job_utils import get_result_df\n","\n","\n","print(f\"\"\"Feathr version: {feathr.__version__}\n","Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ab35fa01-b392-457e-8fde-7e445a3c39b5","showTitle":false,"title":""}},"source":["## 2. Create Shareable Features with Feathr Feature Definition Configs\n","\n","In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n","Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"09f93a9f-7b33-4d91-8f31-ee3b20991696","showTitle":false,"title":""}},"outputs":[],"source":["RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n","PROJECT_NAME = \"feathr_getting_started\"\n","\n","REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n","\n","# Use a databricks cluster\n","SPARK_CLUSTER = \"databricks\"\n","\n","# Databricks file system path\n","DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"41d3648a-9bc9-40dc-90da-bc82b21ef9b3","showTitle":false,"title":""}},"source":["In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec.\n","\n","Note: When submitting jobs, Databricks recommend to use new clusters for greater reliability. If you want to use an existing all-purpose cluster, you may set\n","`existing_cluster_id': ctx.tags().get('clusterId').get()` to the `databricks_config`, replacing `new_cluster` config values."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"331753d6-1850-47b5-ad97-84b7c01d79d1","showTitle":false,"title":""}},"outputs":[],"source":["# Redis credential\n","os.environ['REDIS_PASSWORD'] = REDIS_KEY\n","\n","# Setup databricks env configs\n","ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n","databricks_config = {\n"," 'run_name': \"FEATHR_FILL_IN\",\n"," # To use an existing all-purpose cluster:\n"," # 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n"," # To use a new job cluster:\n"," 'new_cluster': {\n"," 'spark_version': \"11.2.x-scala2.12\",\n"," 'node_type_id': \"Standard_D3_v2\",\n"," 'num_workers':1,\n"," 'spark_conf': {\n"," 'FEATHR_FILL_IN': \"FEATHR_FILL_IN\",\n"," # Exclude conflicting packages if use feathr <= v0.8.0:\n"," 'spark.jars.excludes': \"commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api\",\n"," },\n"," },\n"," 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n"," 'spark_jar_task': {\n"," 'main_class_name': \"FEATHR_FILL_IN\",\n"," 'parameters': [\"FEATHR_FILL_IN\"],\n"," },\n","}\n","os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n","os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n","os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n","os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee","showTitle":false,"title":""}},"source":["### Configurations\n","\n","Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8cd64e3a-376c-48e6-ba41-5197f3591d48","showTitle":false,"title":""}},"outputs":[],"source":["config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n","\n","with open(config_path, 'r') as f: \n"," print(f.read())"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"58d22dc1-7590-494d-94ca-3e2488c31c8e","showTitle":false,"title":""}},"source":["All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3fef7f2f-df19-4f53-90a5-ff7999ed983d","showTitle":false,"title":""}},"source":["### Initialize Feathr Client"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9713a2df-c7b2-4562-88b0-b7acce3cc43a","showTitle":false,"title":""}},"outputs":[],"source":["client = FeathrClient(config_path=config_path)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c3b64bda-d42c-4a64-b976-0fb604cf38c5","showTitle":false,"title":""}},"source":["### View the NYC taxi fare dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c4ccd7b3-298a-4e5a-8eec-b7e309db393e","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n","\n","# Download the data file\n","df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n","df_raw.limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7430c942-64e5-4b70-b823-16ce1d1b3cee","showTitle":false,"title":""}},"source":["### Defining features with Feathr\n","\n","In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n","\n","* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n","* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n","* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n","\n","Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n","\n","There are two types of features -- anchored features and derivated features:\n","\n","* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n","* **Derived features**: Features that are computed on top of other features.\n","\n","#### Define anchored features\n","\n","A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"75b8d2ed-84df-4446-ae07-5f715434f3ea","showTitle":false,"title":""}},"outputs":[],"source":["TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n","TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"93abbcc2-562b-47e4-ad4c-1fedd7cc64df","showTitle":false,"title":""}},"outputs":[],"source":["# We define f_trip_distance and f_trip_time_duration features separately\n","# so that we can reuse them later for the derived features.\n","f_trip_distance = Feature(\n"," name=\"f_trip_distance\",\n"," feature_type=FLOAT,\n"," transform=\"trip_distance\",\n",")\n","f_trip_time_duration = Feature(\n"," name=\"f_trip_time_duration\",\n"," feature_type=FLOAT,\n"," transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",")\n","\n","features = [\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," Feature(\n"," name=\"f_is_long_trip_distance\",\n"," feature_type=BOOLEAN,\n"," transform=\"trip_distance > 30.0\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_week\",\n"," feature_type=INT32,\n"," transform=\"dayofweek(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_month\",\n"," feature_type=INT32,\n"," transform=\"dayofmonth(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_hour_of_day\",\n"," feature_type=INT32,\n"," transform=\"hour(lpep_dropoff_datetime)\",\n"," ),\n","]\n","\n","# After you have defined features, bring them together to build the anchor to the source.\n","feature_anchor = FeatureAnchor(\n"," name=\"feature_anchor\",\n"," source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n"," features=features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"728d2d5f-c11f-4941-bdc5-48507f5749f1","showTitle":false,"title":""}},"source":["We can define the source with a preprocessing python function."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3cc59a0e-a41b-480e-a84e-ca5443d63143","showTitle":false,"title":""}},"outputs":[],"source":["def preprocessing(df: DataFrame) -> DataFrame:\n"," import pyspark.sql.functions as F\n"," df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n"," return df\n","\n","batch_source = HdfsSource(\n"," name=\"nycTaxiBatchSource\",\n"," path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," preprocessing=preprocessing,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"46f863c4-bb81-434a-a448-6b585031a221","showTitle":false,"title":""}},"source":["For the features with aggregation, the supported functions are as follows:\n","\n","| Aggregation Function | Input Type | Description |\n","| --- | --- | --- |\n","|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n","|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n","|LATEST| Any |Returns the latest not-null values from within the defined time window |"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"a373ecbe-a040-4cd3-9d87-0d5f4c5ba553","showTitle":false,"title":""}},"outputs":[],"source":["agg_key = TypedKey(\n"," key_column=\"DOLocationID\",\n"," key_column_type=ValueType.INT32,\n"," description=\"location id in NYC\",\n"," full_name=\"nyc_taxi.location_id\",\n",")\n","\n","agg_window = \"90d\"\n","\n","# Anchored features with aggregations\n","agg_features = [\n"," Feature(\n"," name=\"f_location_avg_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"AVG\",\n"," window=agg_window,\n"," ),\n"," ),\n"," Feature(\n"," name=\"f_location_max_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"MAX\",\n"," window=agg_window,\n"," ),\n"," ),\n","]\n","\n","agg_feature_anchor = FeatureAnchor(\n"," name=\"agg_feature_anchor\",\n"," source=batch_source, # External data source for feature. Typically a data table.\n"," features=agg_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"149f85e2-fa3c-4895-b0c5-de5543ca9b6d","showTitle":false,"title":""}},"source":["#### Define derived features\n","\n","We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"05633bc3-9118-449b-9562-45fc437576c2","showTitle":false,"title":""}},"outputs":[],"source":["derived_features = [\n"," DerivedFeature(\n"," name=\"f_trip_time_distance\",\n"," feature_type=FLOAT,\n"," input_features=[\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," ],\n"," transform=\"f_trip_distance / f_trip_time_duration\",\n"," )\n","]"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ad102c45-586d-468c-85f0-9454401ef10b","showTitle":false,"title":""}},"source":["### Build features\n","\n","Finally, we build the features."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"91bb5ebb-87e4-470b-b8eb-1c89b351740e","showTitle":false,"title":""}},"outputs":[],"source":["client.build_features(\n"," anchor_list=[feature_anchor, agg_feature_anchor],\n"," derived_feature_list=derived_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"632d5f46-f9e2-41a8-aab7-34f75206e2aa","showTitle":false,"title":""}},"source":["## 3. Create Training Data Using Point-in-Time Correct Feature Join\n","\n","After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n","\n","To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n","what features and how these features should be joined to the observation data. \n","\n","To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"02feabc9-2f2f-43e8-898d-b28082798e98","showTitle":false,"title":""}},"outputs":[],"source":["feature_names = [feature.name for feature in features + agg_features + derived_features]\n","feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FORMAT = \"parquet\"\n","offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"67e81466-c736-47ba-b122-e640642c01cf","showTitle":false,"title":""}},"outputs":[],"source":["# Features that we want to request. Can use a subset of features\n","query = FeatureQuery(\n"," feature_list=feature_names,\n"," key=agg_key,\n",")\n","settings = ObservationSettings(\n"," observation_path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")\n","client.get_offline_features(\n"," observation_settings=settings,\n"," feature_query=query,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n"," execution_configurations=SparkExecutionConfiguration({\n"," \"spark.feathr.outputFormat\": DATA_FORMAT,\n"," }),\n"," output_path=offline_features_path,\n",")\n","\n","client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9871af55-25eb-41ee-a58a-fda74b1a174e","showTitle":false,"title":""}},"outputs":[],"source":["# Show feature results\n","df = get_result_df(\n"," spark=spark,\n"," client=client,\n"," data_format=\"parquet\",\n"," res_url=offline_features_path,\n",")\n","df.select(feature_names).limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f","showTitle":false,"title":""}},"source":["## 4. Train and Evaluate a Prediction Model\n","\n","After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n","\n","Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5a226026-1c7b-48db-8f91-88d5c2ddf023","showTitle":false,"title":""}},"source":["### Load Train and Test Data from the Offline Feature Values"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bd2cdc83-0920-46e8-9454-e5e6e7832ce0","showTitle":false,"title":""}},"outputs":[],"source":["# Train / test split\n","train_df, test_df = (\n"," df # Dataframe that we generated from get_offline_features call.\n"," .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n"," .where(F.col(\"f_trip_time_duration\") > 0)\n"," .fillna(0)\n"," .randomSplit([0.8, 0.2])\n",")\n","\n","print(f\"Num train samples: {train_df.count()}\")\n","print(f\"Num test samples: {test_df.count()}\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd","showTitle":false,"title":""}},"source":["### Build a ML Pipeline\n","\n","Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a254361-63e9-45b2-8c19-40549762eacb","showTitle":false,"title":""}},"outputs":[],"source":["# Generate a feature vector column for SparkML\n","vector_assembler = VectorAssembler(\n"," inputCols=[x for x in df.columns if x in feature_names],\n"," outputCol=\"features\",\n",")\n","\n","# Define a model\n","gbt = GBTRegressor(\n"," featuresCol=\"features\",\n"," maxIter=100,\n"," maxDepth=5,\n"," maxBins=16,\n",")\n","\n","# Create a ML pipeline\n","ml_pipeline = Pipeline(stages=[\n"," vector_assembler,\n"," gbt,\n","])"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bef93538-9591-4247-97b6-289d2055b7b1","showTitle":false,"title":""}},"source":["### Train and Evaluate the Model"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"0c3d5f35-11a3-4644-9992-5860169d8302","showTitle":false,"title":""}},"outputs":[],"source":["# Train a model\n","model = ml_pipeline.fit(train_df)\n","\n","# Make predictions\n","predictions = model.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1f9b584c-6228-4a02-a6c3-9b8dd2b78091","showTitle":false,"title":""}},"outputs":[],"source":["# Evaluate\n","evaluator = RegressionEvaluator(\n"," labelCol=\"label\",\n"," predictionCol=\"prediction\",\n",")\n","\n","rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n","mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n","print(f\"RMSE: {rmse}\\nMAE: {mae}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"25c33abd-6e87-437d-a6a1-86435f065a1e","showTitle":false,"title":""}},"outputs":[],"source":["# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n","predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n","\n","predictions_pdf.plot(\n"," x=\"index\",\n"," y=[\"label\", \"prediction\"],\n"," style=['-', ':'],\n"," figsize=(20, 10),\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"664d78cc-4a92-430c-9e05-565ba904558e","showTitle":false,"title":""}},"outputs":[],"source":["predictions_pdf.plot.scatter(\n"," x=\"label\",\n"," y=\"prediction\",\n"," xlim=(0, 100),\n"," ylim=(0, 100),\n"," figsize=(10, 10),\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8a56d165-c813-4ce0-8ae6-9f4d313c463d","showTitle":false,"title":""}},"source":["## 5. Materialize Feature Values for Online Scoring\n","\n","While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n","\n","Note, only the features anchored to offline data source can be materialized."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"751fa72e-8f94-40a1-994e-3e8315b51d37","showTitle":false,"title":""}},"outputs":[],"source":["materialized_feature_names = [feature.name for feature in agg_features]\n","materialized_feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4d4699ed-42e6-408f-903d-2f799284f4b6","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n","\n"," # Get the last date from the dataset\n"," backfill_timestamp = (\n"," df_raw\n"," .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n"," .agg({TIMESTAMP_COL: \"max\"})\n"," .collect()[0][0]\n"," )\n","\n"," # Time range to materialize\n"," backfill_time = BackfillTime(\n"," start=backfill_timestamp,\n"," end=backfill_timestamp,\n"," step=timedelta(days=1),\n"," )\n","\n"," # Destinations:\n"," # For online store,\n"," redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n","\n"," # For offline store,\n"," # adls_sink = HdfsSink(output_path=)\n","\n"," settings = MaterializationSettings(\n"," name=FEATURE_TABLE_NAME + \".job\", # job name\n"," backfill_time=backfill_time,\n"," sinks=[redis_sink], # or adls_sink\n"," feature_names=materialized_feature_names,\n"," )\n","\n"," client.materialize_features(\n"," settings=settings,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n"," )\n","\n"," client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9","showTitle":false,"title":""}},"source":["Now, you can retrieve features for online scoring as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"424bc9eb-a47f-4b46-be69-8218d55e66ad","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," # Note, to get a single key, you may use client.get_online_features instead\n"," materialized_feature_values = client.multi_get_online_features(\n"," feature_table=FEATURE_TABLE_NAME,\n"," keys=[\"239\", \"265\"],\n"," feature_names=materialized_feature_names,\n"," )\n"," materialized_feature_values"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3596dc71-a363-4b6a-a169-215c89978558","showTitle":false,"title":""}},"source":["## Cleanup"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"b5fb292e-bbb6-4dd7-8e79-c62d9533e820","showTitle":false,"title":""}},"outputs":[],"source":["# Remove temporary files\n","dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"databricks_quickstart_nyc_taxi_demo","notebookOrigID":2365994027381987,"widgets":{"REDIS_KEY":{"currentValue":"","nuid":"d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca","widgetInfo":{"defaultValue":"","label":null,"name":"REDIS_KEY","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}},"RESOURCE_PREFIX":{"currentValue":"","nuid":"87a26035-86fc-4dbd-8dd0-dc546c1c63c1","widgetInfo":{"defaultValue":"","label":null,"name":"RESOURCE_PREFIX","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}}}},"kernelspec":{"display_name":"Python 3.10.8 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.8"},"vscode":{"interpreter":{"hash":"b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"}}},"nbformat":4,"nbformat_minor":0}
+{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"843d3142-24ca-4bd1-9e31-b55163804fe3","showTitle":false,"title":""}},"outputs":[],"source":["dbutils.widgets.text(\"RESOURCE_PREFIX\", \"\")\n","dbutils.widgets.text(\"REDIS_KEY\", \"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"384e5e16-7213-4186-9d04-09d03b155534","showTitle":false,"title":""}},"source":["# Feathr Feature Store on Databricks Demo Notebook\n","\n","This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).\n","\n","This notebook is specifically written for Databricks and is relying on some of the Databricks packages such as `dbutils`. The intention here is to provide a \"one click run\" example with minimum configuration. For example:\n","- This notebook skips feature registry which requires running Azure Purview. \n","- To make the online feature query work, you will need to configure the Redis endpoint. \n","\n","The full-fledged notebook can be found from [here](https://github.com/feathr-ai/feathr/blob/main/docs/samples/nyc_taxi_demo.ipynb)."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c2ce58c7-9263-469a-bbb7-43364ddb07b8","showTitle":false,"title":""}},"source":["## Prerequisite\n","\n","To use feathr materialization for online scoring with Redis cache, you may deploy a Redis cluster and set `RESOURCE_PREFIX` and `REDIS_KEY` via Databricks widgets. Note that the deployed Redis host address should be `{RESOURCE_PREFIX}redis.redis.cache.windows.net`. More details about how to deploy the Redis cluster can be found [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html#configurure-redis-cluster).\n","\n","To run this notebook, you'll need to install `feathr` pip package. Here, we install notebook-scoped library. For details, please see [Azure Databricks dependency management document](https://learn.microsoft.com/en-us/azure/databricks/libraries/)."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4609d7ad-ad74-40fc-b97e-f440a0fa0737","showTitle":false,"title":""}},"outputs":[],"source":["# Install feathr from the latest codes in the repo. You may use `pip install feathr` as well.\n","!pip install \"git+https://github.com/feathr-ai/feathr#subdirectory=feathr_project\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c81fa80c-bca6-4ae5-84ad-659a036977bd","showTitle":false,"title":""}},"source":["## Notebook Steps\n","\n","This tutorial demonstrates the key capabilities of Feathr, including:\n","\n","1. Install Feathr and necessary dependencies.\n","1. Create shareable features with Feathr feature definition configs.\n","1. Create training data using point-in-time correct feature join\n","1. Train and evaluate a prediction model.\n","1. Materialize feature values for online scoring.\n","\n","The overall data flow is as follows:\n","\n",""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"80223a02-631c-40c8-91b3-a037249ffff9","showTitle":false,"title":""}},"outputs":[],"source":["from datetime import datetime, timedelta\n","import glob\n","import json\n","from math import sqrt\n","import os\n","from pathlib import Path\n","import requests\n","from tempfile import TemporaryDirectory\n","\n","from azure.identity import AzureCliCredential, DefaultAzureCredential \n","from azure.keyvault.secrets import SecretClient\n","import pandas as pd\n","from pyspark.ml import Pipeline\n","from pyspark.ml.evaluation import RegressionEvaluator\n","from pyspark.ml.feature import VectorAssembler\n","from pyspark.ml.regression import GBTRegressor\n","from pyspark.sql import DataFrame, SparkSession\n","import pyspark.sql.functions as F\n","\n","import feathr\n","from feathr import (\n"," FeathrClient,\n"," # Feature data types\n"," BOOLEAN, FLOAT, INT32, ValueType,\n"," # Feature data sources\n"," INPUT_CONTEXT, HdfsSource,\n"," # Feature aggregations\n"," TypedKey, WindowAggTransformation,\n"," # Feature types and anchor\n"," DerivedFeature, Feature, FeatureAnchor,\n"," # Materialization\n"," BackfillTime, MaterializationSettings, RedisSink,\n"," # Offline feature computation\n"," FeatureQuery, ObservationSettings,\n",")\n","from feathr.datasets import nyc_taxi\n","from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n","from feathr.utils.config import generate_config\n","from feathr.utils.job_utils import get_result_df\n","\n","\n","print(f\"\"\"Feathr version: {feathr.__version__}\n","Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ab35fa01-b392-457e-8fde-7e445a3c39b5","showTitle":false,"title":""}},"source":["## 2. Create Shareable Features with Feathr Feature Definition Configs\n","\n","In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n","Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"09f93a9f-7b33-4d91-8f31-ee3b20991696","showTitle":false,"title":""}},"outputs":[],"source":["RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n","PROJECT_NAME = \"feathr_getting_started\"\n","\n","REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n","\n","# Use a databricks cluster\n","SPARK_CLUSTER = \"databricks\"\n","\n","# Databricks file system path\n","DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"41d3648a-9bc9-40dc-90da-bc82b21ef9b3","showTitle":false,"title":""}},"source":["In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec.\n","\n","Note: When submitting jobs, Databricks recommend to use new clusters for greater reliability. If you want to use an existing all-purpose cluster, you may set\n","`existing_cluster_id': ctx.tags().get('clusterId').get()` to the `databricks_config`, replacing `new_cluster` config values."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"331753d6-1850-47b5-ad97-84b7c01d79d1","showTitle":false,"title":""}},"outputs":[],"source":["# Redis credential\n","os.environ['REDIS_PASSWORD'] = REDIS_KEY\n","\n","# Setup databricks env configs\n","ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n","databricks_config = {\n"," 'run_name': \"FEATHR_FILL_IN\",\n"," # To use an existing all-purpose cluster:\n"," # 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n"," # To use a new job cluster:\n"," 'new_cluster': {\n"," 'spark_version': \"11.2.x-scala2.12\",\n"," 'node_type_id': \"Standard_D3_v2\",\n"," 'num_workers':1,\n"," 'spark_conf': {\n"," 'FEATHR_FILL_IN': \"FEATHR_FILL_IN\",\n"," # Exclude conflicting packages if use feathr <= v0.8.0:\n"," 'spark.jars.excludes': \"commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api\",\n"," },\n"," },\n"," 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n"," 'spark_jar_task': {\n"," 'main_class_name': \"FEATHR_FILL_IN\",\n"," 'parameters': [\"FEATHR_FILL_IN\"],\n"," },\n","}\n","os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n","os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n","os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n","os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee","showTitle":false,"title":""}},"source":["### Configurations\n","\n","Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8cd64e3a-376c-48e6-ba41-5197f3591d48","showTitle":false,"title":""}},"outputs":[],"source":["config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n","\n","with open(config_path, 'r') as f: \n"," print(f.read())"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"58d22dc1-7590-494d-94ca-3e2488c31c8e","showTitle":false,"title":""}},"source":["All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3fef7f2f-df19-4f53-90a5-ff7999ed983d","showTitle":false,"title":""}},"source":["### Initialize Feathr Client"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9713a2df-c7b2-4562-88b0-b7acce3cc43a","showTitle":false,"title":""}},"outputs":[],"source":["client = FeathrClient(config_path=config_path)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c3b64bda-d42c-4a64-b976-0fb604cf38c5","showTitle":false,"title":""}},"source":["### View the NYC taxi fare dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c4ccd7b3-298a-4e5a-8eec-b7e309db393e","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n","\n","# Download the data file\n","df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n","df_raw.limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7430c942-64e5-4b70-b823-16ce1d1b3cee","showTitle":false,"title":""}},"source":["### Defining features with Feathr\n","\n","In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n","\n","* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n","* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n","* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n","\n","Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n","\n","There are two types of features -- anchored features and derivated features:\n","\n","* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n","* **Derived features**: Features that are computed on top of other features.\n","\n","#### Define anchored features\n","\n","A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"75b8d2ed-84df-4446-ae07-5f715434f3ea","showTitle":false,"title":""}},"outputs":[],"source":["TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n","TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"93abbcc2-562b-47e4-ad4c-1fedd7cc64df","showTitle":false,"title":""}},"outputs":[],"source":["# We define f_trip_distance and f_trip_time_duration features separately\n","# so that we can reuse them later for the derived features.\n","f_trip_distance = Feature(\n"," name=\"f_trip_distance\",\n"," feature_type=FLOAT,\n"," transform=\"trip_distance\",\n",")\n","f_trip_time_duration = Feature(\n"," name=\"f_trip_time_duration\",\n"," feature_type=FLOAT,\n"," transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",")\n","\n","features = [\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," Feature(\n"," name=\"f_is_long_trip_distance\",\n"," feature_type=BOOLEAN,\n"," transform=\"trip_distance > 30.0\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_week\",\n"," feature_type=INT32,\n"," transform=\"dayofweek(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_month\",\n"," feature_type=INT32,\n"," transform=\"dayofmonth(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_hour_of_day\",\n"," feature_type=INT32,\n"," transform=\"hour(lpep_dropoff_datetime)\",\n"," ),\n","]\n","\n","# After you have defined features, bring them together to build the anchor to the source.\n","feature_anchor = FeatureAnchor(\n"," name=\"feature_anchor\",\n"," source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n"," features=features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"728d2d5f-c11f-4941-bdc5-48507f5749f1","showTitle":false,"title":""}},"source":["We can define the source with a preprocessing python function."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3cc59a0e-a41b-480e-a84e-ca5443d63143","showTitle":false,"title":""}},"outputs":[],"source":["def preprocessing(df: DataFrame) -> DataFrame:\n"," import pyspark.sql.functions as F\n"," df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n"," return df\n","\n","batch_source = HdfsSource(\n"," name=\"nycTaxiBatchSource\",\n"," path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," preprocessing=preprocessing,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"46f863c4-bb81-434a-a448-6b585031a221","showTitle":false,"title":""}},"source":["For the features with aggregation, the supported functions are as follows:\n","\n","| Aggregation Function | Input Type | Description |\n","| --- | --- | --- |\n","|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n","|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n","|LATEST| Any |Returns the latest not-null values from within the defined time window |"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"a373ecbe-a040-4cd3-9d87-0d5f4c5ba553","showTitle":false,"title":""}},"outputs":[],"source":["agg_key = TypedKey(\n"," key_column=\"DOLocationID\",\n"," key_column_type=ValueType.INT32,\n"," description=\"location id in NYC\",\n"," full_name=\"nyc_taxi.location_id\",\n",")\n","\n","agg_window = \"90d\"\n","\n","# Anchored features with aggregations\n","agg_features = [\n"," Feature(\n"," name=\"f_location_avg_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"AVG\",\n"," window=agg_window,\n"," ),\n"," ),\n"," Feature(\n"," name=\"f_location_max_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"MAX\",\n"," window=agg_window,\n"," ),\n"," ),\n","]\n","\n","agg_feature_anchor = FeatureAnchor(\n"," name=\"agg_feature_anchor\",\n"," source=batch_source, # External data source for feature. Typically a data table.\n"," features=agg_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"149f85e2-fa3c-4895-b0c5-de5543ca9b6d","showTitle":false,"title":""}},"source":["#### Define derived features\n","\n","We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"05633bc3-9118-449b-9562-45fc437576c2","showTitle":false,"title":""}},"outputs":[],"source":["derived_features = [\n"," DerivedFeature(\n"," name=\"f_trip_time_distance\",\n"," feature_type=FLOAT,\n"," input_features=[\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," ],\n"," transform=\"f_trip_distance / f_trip_time_duration\",\n"," )\n","]"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ad102c45-586d-468c-85f0-9454401ef10b","showTitle":false,"title":""}},"source":["### Build features\n","\n","Finally, we build the features."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"91bb5ebb-87e4-470b-b8eb-1c89b351740e","showTitle":false,"title":""}},"outputs":[],"source":["client.build_features(\n"," anchor_list=[feature_anchor, agg_feature_anchor],\n"," derived_feature_list=derived_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"632d5f46-f9e2-41a8-aab7-34f75206e2aa","showTitle":false,"title":""}},"source":["## 3. Create Training Data Using Point-in-Time Correct Feature Join\n","\n","After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n","\n","To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n","what features and how these features should be joined to the observation data. \n","\n","To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"02feabc9-2f2f-43e8-898d-b28082798e98","showTitle":false,"title":""}},"outputs":[],"source":["feature_names = [feature.name for feature in features + agg_features + derived_features]\n","feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FORMAT = \"parquet\"\n","offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"67e81466-c736-47ba-b122-e640642c01cf","showTitle":false,"title":""}},"outputs":[],"source":["# Features that we want to request. Can use a subset of features\n","query = FeatureQuery(\n"," feature_list=feature_names,\n"," key=agg_key,\n",")\n","settings = ObservationSettings(\n"," observation_path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")\n","client.get_offline_features(\n"," observation_settings=settings,\n"," feature_query=query,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n"," execution_configurations=SparkExecutionConfiguration({\n"," \"spark.feathr.outputFormat\": DATA_FORMAT,\n"," }),\n"," output_path=offline_features_path,\n",")\n","\n","client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9871af55-25eb-41ee-a58a-fda74b1a174e","showTitle":false,"title":""}},"outputs":[],"source":["# Show feature results\n","df = get_result_df(\n"," spark=spark,\n"," client=client,\n"," data_format=\"parquet\",\n"," res_url=offline_features_path,\n",")\n","df.select(feature_names).limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f","showTitle":false,"title":""}},"source":["## 4. Train and Evaluate a Prediction Model\n","\n","After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n","\n","Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5a226026-1c7b-48db-8f91-88d5c2ddf023","showTitle":false,"title":""}},"source":["### Load Train and Test Data from the Offline Feature Values"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bd2cdc83-0920-46e8-9454-e5e6e7832ce0","showTitle":false,"title":""}},"outputs":[],"source":["# Train / test split\n","train_df, test_df = (\n"," df # Dataframe that we generated from get_offline_features call.\n"," .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n"," .where(F.col(\"f_trip_time_duration\") > 0)\n"," .fillna(0)\n"," .randomSplit([0.8, 0.2])\n",")\n","\n","print(f\"Num train samples: {train_df.count()}\")\n","print(f\"Num test samples: {test_df.count()}\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd","showTitle":false,"title":""}},"source":["### Build a ML Pipeline\n","\n","Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a254361-63e9-45b2-8c19-40549762eacb","showTitle":false,"title":""}},"outputs":[],"source":["# Generate a feature vector column for SparkML\n","vector_assembler = VectorAssembler(\n"," inputCols=[x for x in df.columns if x in feature_names],\n"," outputCol=\"features\",\n",")\n","\n","# Define a model\n","gbt = GBTRegressor(\n"," featuresCol=\"features\",\n"," maxIter=100,\n"," maxDepth=5,\n"," maxBins=16,\n",")\n","\n","# Create a ML pipeline\n","ml_pipeline = Pipeline(stages=[\n"," vector_assembler,\n"," gbt,\n","])"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bef93538-9591-4247-97b6-289d2055b7b1","showTitle":false,"title":""}},"source":["### Train and Evaluate the Model"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"0c3d5f35-11a3-4644-9992-5860169d8302","showTitle":false,"title":""}},"outputs":[],"source":["# Train a model\n","model = ml_pipeline.fit(train_df)\n","\n","# Make predictions\n","predictions = model.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1f9b584c-6228-4a02-a6c3-9b8dd2b78091","showTitle":false,"title":""}},"outputs":[],"source":["# Evaluate\n","evaluator = RegressionEvaluator(\n"," labelCol=\"label\",\n"," predictionCol=\"prediction\",\n",")\n","\n","rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n","mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n","print(f\"RMSE: {rmse}\\nMAE: {mae}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"25c33abd-6e87-437d-a6a1-86435f065a1e","showTitle":false,"title":""}},"outputs":[],"source":["# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n","predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n","\n","predictions_pdf.plot(\n"," x=\"index\",\n"," y=[\"label\", \"prediction\"],\n"," style=['-', ':'],\n"," figsize=(20, 10),\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"664d78cc-4a92-430c-9e05-565ba904558e","showTitle":false,"title":""}},"outputs":[],"source":["predictions_pdf.plot.scatter(\n"," x=\"label\",\n"," y=\"prediction\",\n"," xlim=(0, 100),\n"," ylim=(0, 100),\n"," figsize=(10, 10),\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8a56d165-c813-4ce0-8ae6-9f4d313c463d","showTitle":false,"title":""}},"source":["## 5. Materialize Feature Values for Online Scoring\n","\n","While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n","\n","Note, only the features anchored to offline data source can be materialized."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"751fa72e-8f94-40a1-994e-3e8315b51d37","showTitle":false,"title":""}},"outputs":[],"source":["materialized_feature_names = [feature.name for feature in agg_features]\n","materialized_feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4d4699ed-42e6-408f-903d-2f799284f4b6","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n","\n"," # Get the last date from the dataset\n"," backfill_timestamp = (\n"," df_raw\n"," .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n"," .agg({TIMESTAMP_COL: \"max\"})\n"," .collect()[0][0]\n"," )\n","\n"," # Time range to materialize\n"," backfill_time = BackfillTime(\n"," start=backfill_timestamp,\n"," end=backfill_timestamp,\n"," step=timedelta(days=1),\n"," )\n","\n"," # Destinations:\n"," # For online store,\n"," redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n","\n"," # For offline store,\n"," # adls_sink = HdfsSink(output_path=)\n","\n"," settings = MaterializationSettings(\n"," name=FEATURE_TABLE_NAME + \".job\", # job name\n"," backfill_time=backfill_time,\n"," sinks=[redis_sink], # or adls_sink\n"," feature_names=materialized_feature_names,\n"," )\n","\n"," client.materialize_features(\n"," settings=settings,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n"," )\n","\n"," client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9","showTitle":false,"title":""}},"source":["Now, you can retrieve features for online scoring as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"424bc9eb-a47f-4b46-be69-8218d55e66ad","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," # Note, to get a single key, you may use client.get_online_features instead\n"," materialized_feature_values = client.multi_get_online_features(\n"," feature_table=FEATURE_TABLE_NAME,\n"," keys=[\"239\", \"265\"],\n"," feature_names=materialized_feature_names,\n"," )\n"," materialized_feature_values"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3596dc71-a363-4b6a-a169-215c89978558","showTitle":false,"title":""}},"source":["## Cleanup"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"b5fb292e-bbb6-4dd7-8e79-c62d9533e820","showTitle":false,"title":""}},"outputs":[],"source":["# Remove temporary files\n","dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"databricks_quickstart_nyc_taxi_demo","notebookOrigID":2365994027381987,"widgets":{"REDIS_KEY":{"currentValue":"","nuid":"d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca","widgetInfo":{"defaultValue":"","label":null,"name":"REDIS_KEY","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}},"RESOURCE_PREFIX":{"currentValue":"","nuid":"87a26035-86fc-4dbd-8dd0-dc546c1c63c1","widgetInfo":{"defaultValue":"","label":null,"name":"RESOURCE_PREFIX","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}}}},"kernelspec":{"display_name":"Python 3.10.8 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.8"},"vscode":{"interpreter":{"hash":"b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"}}},"nbformat":4,"nbformat_minor":0}
diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb
deleted file mode 100644
index ffd6e64d8..000000000
--- a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb
+++ /dev/null
@@ -1,1444 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "384e5e16-7213-4186-9d04-09d03b155534",
- "showTitle": false,
- "title": ""
- }
- },
- "source": [
- "# Feathr Feature Store on Databricks Demo Notebook\n",
- "\n",
- "This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. This is a notebook that's specially designed for databricks clusters and is relying on some of the databricks packages such as dbutils.\n",
- "\n",
- "The intent of this notebook is like \"one click run\" without configuring anything, so it has relatively limited capability. \n",
- "\n",
- "- For example, in this notebook there's no feature registry available since that requires running Azure Purview. \n",
- "- Also for online store (Redis), you need to configure the Redis endpoint, otherwise that part will not work. \n",
- "\n",
- "However, the core part of Feathr, especially defining features, get offline features, point-in-time joins etc., should \"just work\". The full-fledged notebook is [located here](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb)."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "# Notebook Steps\n",
- "\n",
- "This tutorial demonstrates the key capabilities of Feathr, including:\n",
- "\n",
- "1. Install and set up Feathr with Azure\n",
- "2. Create shareable features with Feathr feature definition configs.\n",
- "3. Create a training dataset via point-in-time feature join.\n",
- "4. Compute and write features.\n",
- "5. Train a model using these features to predict fares.\n",
- "6. Materialize feature value to online store.\n",
- "7. Fetch feature value in real-time from online store for online scoring.\n",
- "\n",
- "In this tutorial, we use Feathr Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). The feature flow is as below:\n",
- "\n",
- ""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "f00b9d0b-94d1-418f-89b9-25bbacb8b068",
- "showTitle": false,
- "title": ""
- }
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ]
- },
- "metadata": {
- "application/vnd.databricks.v1+output": {
- "arguments": {},
- "data": "",
- "errorSummary": "",
- "errorTraceType": null,
- "metadata": {},
- "type": "ipynbError"
- }
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "! pip install feathr pandavro scikit-learn"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "80223a02-631c-40c8-91b3-a037249ffff9",
- "showTitle": false,
- "title": ""
- }
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ]
- },
- "metadata": {
- "application/vnd.databricks.v1+output": {
- "arguments": {},
- "data": "",
- "errorSummary": "",
- "errorTraceType": null,
- "metadata": {},
- "type": "ipynbError"
- }
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "import glob\n",
- "import os\n",
- "import tempfile\n",
- "from datetime import datetime, timedelta\n",
- "from math import sqrt\n",
- "\n",
- "import pandas as pd\n",
- "import pandavro as pdx\n",
- "from feathr import FeathrClient\n",
- "from feathr import BOOLEAN, FLOAT, INT32, ValueType\n",
- "from feathr import Feature, DerivedFeature, FeatureAnchor\n",
- "from feathr import BackfillTime, MaterializationSettings\n",
- "from feathr import FeatureQuery, ObservationSettings\n",
- "from feathr import RedisSink\n",
- "from feathr import INPUT_CONTEXT, HdfsSource\n",
- "from feathr import WindowAggTransformation\n",
- "from feathr import TypedKey\n",
- "from sklearn.metrics import mean_squared_error\n",
- "from sklearn.model_selection import train_test_split\n",
- "from azure.identity import DefaultAzureCredential\n",
- "from azure.keyvault.secrets import SecretClient\n",
- "import json\n",
- "import requests"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "41d3648a-9bc9-40dc-90da-bc82b21ef9b3",
- "showTitle": false,
- "title": ""
- }
- },
- "source": [
- "Get the required databricks credentials automatically:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "331753d6-1850-47b5-ad97-84b7c01d79d1",
- "showTitle": false,
- "title": ""
- }
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ]
- },
- "metadata": {
- "application/vnd.databricks.v1+output": {
- "arguments": {},
- "data": "",
- "errorSummary": "",
- "errorTraceType": null,
- "metadata": {},
- "type": "ipynbError"
- }
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "# Get current databricks notebook context\n",
- "ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n",
- "host_name = ctx.tags().get(\"browserHostName\").get()\n",
- "host_token = ctx.apiToken().get()\n",
- "cluster_id = ctx.tags().get(\"clusterId\").get()\n",
- "\n",
- "\n",
- "\n",
- "# databricks_config = {'run_name':'FEATHR_FILL_IN','existing_cluster_id':cluster_id,'libraries':[{'jar':'FEATHR_FILL_IN'}],'spark_jar_task':{'main_class_name':'FEATHR_FILL_IN','parameters':['FEATHR_FILL_IN']}}\n",
- "os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + host_name\n",
- "os.environ['spark_config__databricks__config_template']='{\"run_name\":\"FEATHR_FILL_IN\",\"new_cluster\":{\"spark_version\":\"10.4.x-scala2.12\",\"node_type_id\":\"Standard_D3_v2\",\"num_workers\":2,\"spark_conf\":{\"FEATHR_FILL_IN\":\"FEATHR_FILL_IN\"}},\"libraries\":[{\"jar\":\"FEATHR_FILL_IN\"}],\"spark_jar_task\":{\"main_class_name\":\"FEATHR_FILL_IN\",\"parameters\":[\"FEATHR_FILL_IN\"]}}'\n",
- "# os.environ['spark_config__databricks__config_template']=json.dumps(databricks_config)\n",
- "os.environ['spark_config__databricks__work_dir']='dbfs:/feathr_getting_started'\n",
- "os.environ['project_config__project_name']='feathr_getting_started'\n",
- "os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = host_token"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You need to setup the Redis credentials below in order to push features to online store. You can skip this part if you don't have Redis, but there will be failures for `client.materialize_features(settings)` API."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Get redis credentials; This is to parse Redis connection string.\n",
- "redis_port=\"\"\n",
- "redis_host=\"\"\n",
- "redis_password=\"\"\n",
- "redis_ssl=\"\"\n",
- "\n",
- "# Set the resource link\n",
- "os.environ['online_store__redis__host'] = redis_host\n",
- "os.environ['online_store__redis__port'] = redis_port\n",
- "os.environ['online_store__redis__ssl_enabled'] = redis_ssl\n",
- "os.environ['REDIS_PASSWORD']=redis_password"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee",
- "showTitle": false,
- "title": ""
- }
- },
- "source": [
- "Configure required credentials (skip if you don't use those):"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "application/vnd.databricks.v1+cell": {
- "inputWidgets": {},
- "nuid": "8cd64e3a-376c-48e6-ba41-5197f3591d48",
- "showTitle": false,
- "title": ""
- }
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ]
- },
- "metadata": {
- "application/vnd.databricks.v1+output": {
- "arguments": {},
- "data": "",
- "errorSummary": "",
- "errorTraceType": null,
- "metadata": {},
- "type": "ipynbError"
- }
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "import tempfile\n",
- "yaml_config = \"\"\"\n",
- "# Please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n",
- "api_version: 1\n",
- "project_config:\n",
- " project_name: 'feathr_getting_started2'\n",
- " required_environment_variables:\n",
- " - 'REDIS_PASSWORD'\n",
- "offline_store:\n",
- " adls:\n",
- " adls_enabled: true\n",
- " wasb:\n",
- " wasb_enabled: true\n",
- " s3:\n",
- " s3_enabled: false\n",
- " s3_endpoint: ''\n",
- " jdbc:\n",
- " jdbc_enabled: false\n",
- " jdbc_database: ''\n",
- " jdbc_table: ''\n",
- " snowflake:\n",
- " snowflake_enabled: false\n",
- " url: \""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"80223a02-631c-40c8-91b3-a037249ffff9","showTitle":false,"title":""}},"outputs":[],"source":["from datetime import datetime, timedelta\n","import glob\n","import json\n","from math import sqrt\n","import os\n","from pathlib import Path\n","import requests\n","from tempfile import TemporaryDirectory\n","\n","from azure.identity import AzureCliCredential, DefaultAzureCredential \n","from azure.keyvault.secrets import SecretClient\n","import pandas as pd\n","from pyspark.ml import Pipeline\n","from pyspark.ml.evaluation import RegressionEvaluator\n","from pyspark.ml.feature import VectorAssembler\n","from pyspark.ml.regression import GBTRegressor\n","from pyspark.sql import DataFrame, SparkSession\n","import pyspark.sql.functions as F\n","\n","import feathr\n","from feathr import (\n"," FeathrClient,\n"," # Feature data types\n"," BOOLEAN, FLOAT, INT32, ValueType,\n"," # Feature data sources\n"," INPUT_CONTEXT, HdfsSource,\n"," # Feature aggregations\n"," TypedKey, WindowAggTransformation,\n"," # Feature types and anchor\n"," DerivedFeature, Feature, FeatureAnchor,\n"," # Materialization\n"," BackfillTime, MaterializationSettings, RedisSink,\n"," # Offline feature computation\n"," FeatureQuery, ObservationSettings,\n",")\n","from feathr.datasets import nyc_taxi\n","from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n","from feathr.utils.config import generate_config\n","from feathr.utils.job_utils import get_result_df\n","\n","\n","print(f\"\"\"Feathr version: {feathr.__version__}\n","Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ab35fa01-b392-457e-8fde-7e445a3c39b5","showTitle":false,"title":""}},"source":["## 2. Create Shareable Features with Feathr Feature Definition Configs\n","\n","In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n","Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"09f93a9f-7b33-4d91-8f31-ee3b20991696","showTitle":false,"title":""}},"outputs":[],"source":["RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n","PROJECT_NAME = \"feathr_getting_started\"\n","\n","REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n","\n","# Use a databricks cluster\n","SPARK_CLUSTER = \"databricks\"\n","\n","# Databricks file system path\n","DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"41d3648a-9bc9-40dc-90da-bc82b21ef9b3","showTitle":false,"title":""}},"source":["In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec.\n","\n","Note: When submitting jobs, Databricks recommend to use new clusters for greater reliability. If you want to use an existing all-purpose cluster, you may set\n","`existing_cluster_id': ctx.tags().get('clusterId').get()` to the `databricks_config`, replacing `new_cluster` config values."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"331753d6-1850-47b5-ad97-84b7c01d79d1","showTitle":false,"title":""}},"outputs":[],"source":["# Redis credential\n","os.environ['REDIS_PASSWORD'] = REDIS_KEY\n","\n","# Setup databricks env configs\n","ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n","databricks_config = {\n"," 'run_name': \"FEATHR_FILL_IN\",\n"," # To use an existing all-purpose cluster:\n"," # 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n"," # To use a new job cluster:\n"," 'new_cluster': {\n"," 'spark_version': \"11.2.x-scala2.12\",\n"," 'node_type_id': \"Standard_D3_v2\",\n"," 'num_workers':1,\n"," 'spark_conf': {\n"," 'FEATHR_FILL_IN': \"FEATHR_FILL_IN\",\n"," # Exclude conflicting packages if use feathr <= v0.8.0:\n"," 'spark.jars.excludes': \"commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api\",\n"," },\n"," },\n"," 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n"," 'spark_jar_task': {\n"," 'main_class_name': \"FEATHR_FILL_IN\",\n"," 'parameters': [\"FEATHR_FILL_IN\"],\n"," },\n","}\n","os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n","os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n","os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n","os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee","showTitle":false,"title":""}},"source":["### Configurations\n","\n","Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8cd64e3a-376c-48e6-ba41-5197f3591d48","showTitle":false,"title":""}},"outputs":[],"source":["config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n","\n","with open(config_path, 'r') as f: \n"," print(f.read())"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"58d22dc1-7590-494d-94ca-3e2488c31c8e","showTitle":false,"title":""}},"source":["All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3fef7f2f-df19-4f53-90a5-ff7999ed983d","showTitle":false,"title":""}},"source":["### Initialize Feathr Client"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9713a2df-c7b2-4562-88b0-b7acce3cc43a","showTitle":false,"title":""}},"outputs":[],"source":["client = FeathrClient(config_path=config_path)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c3b64bda-d42c-4a64-b976-0fb604cf38c5","showTitle":false,"title":""}},"source":["### View the NYC taxi fare dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c4ccd7b3-298a-4e5a-8eec-b7e309db393e","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n","\n","# Download the data file\n","df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n","df_raw.limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7430c942-64e5-4b70-b823-16ce1d1b3cee","showTitle":false,"title":""}},"source":["### Defining features with Feathr\n","\n","In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n","\n","* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n","* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n","* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n","\n","Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n","\n","There are two types of features -- anchored features and derivated features:\n","\n","* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n","* **Derived features**: Features that are computed on top of other features.\n","\n","#### Define anchored features\n","\n","A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"75b8d2ed-84df-4446-ae07-5f715434f3ea","showTitle":false,"title":""}},"outputs":[],"source":["TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n","TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"93abbcc2-562b-47e4-ad4c-1fedd7cc64df","showTitle":false,"title":""}},"outputs":[],"source":["# We define f_trip_distance and f_trip_time_duration features separately\n","# so that we can reuse them later for the derived features.\n","f_trip_distance = Feature(\n"," name=\"f_trip_distance\",\n"," feature_type=FLOAT,\n"," transform=\"trip_distance\",\n",")\n","f_trip_time_duration = Feature(\n"," name=\"f_trip_time_duration\",\n"," feature_type=FLOAT,\n"," transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",")\n","\n","features = [\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," Feature(\n"," name=\"f_is_long_trip_distance\",\n"," feature_type=BOOLEAN,\n"," transform=\"trip_distance > 30.0\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_week\",\n"," feature_type=INT32,\n"," transform=\"dayofweek(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_month\",\n"," feature_type=INT32,\n"," transform=\"dayofmonth(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_hour_of_day\",\n"," feature_type=INT32,\n"," transform=\"hour(lpep_dropoff_datetime)\",\n"," ),\n","]\n","\n","# After you have defined features, bring them together to build the anchor to the source.\n","feature_anchor = FeatureAnchor(\n"," name=\"feature_anchor\",\n"," source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n"," features=features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"728d2d5f-c11f-4941-bdc5-48507f5749f1","showTitle":false,"title":""}},"source":["We can define the source with a preprocessing python function."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3cc59a0e-a41b-480e-a84e-ca5443d63143","showTitle":false,"title":""}},"outputs":[],"source":["def preprocessing(df: DataFrame) -> DataFrame:\n"," import pyspark.sql.functions as F\n"," df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n"," return df\n","\n","batch_source = HdfsSource(\n"," name=\"nycTaxiBatchSource\",\n"," path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," preprocessing=preprocessing,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"46f863c4-bb81-434a-a448-6b585031a221","showTitle":false,"title":""}},"source":["For the features with aggregation, the supported functions are as follows:\n","\n","| Aggregation Function | Input Type | Description |\n","| --- | --- | --- |\n","|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n","|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n","|LATEST| Any |Returns the latest not-null values from within the defined time window |"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"a373ecbe-a040-4cd3-9d87-0d5f4c5ba553","showTitle":false,"title":""}},"outputs":[],"source":["agg_key = TypedKey(\n"," key_column=\"DOLocationID\",\n"," key_column_type=ValueType.INT32,\n"," description=\"location id in NYC\",\n"," full_name=\"nyc_taxi.location_id\",\n",")\n","\n","agg_window = \"90d\"\n","\n","# Anchored features with aggregations\n","agg_features = [\n"," Feature(\n"," name=\"f_location_avg_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"AVG\",\n"," window=agg_window,\n"," ),\n"," ),\n"," Feature(\n"," name=\"f_location_max_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"MAX\",\n"," window=agg_window,\n"," ),\n"," ),\n","]\n","\n","agg_feature_anchor = FeatureAnchor(\n"," name=\"agg_feature_anchor\",\n"," source=batch_source, # External data source for feature. Typically a data table.\n"," features=agg_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"149f85e2-fa3c-4895-b0c5-de5543ca9b6d","showTitle":false,"title":""}},"source":["#### Define derived features\n","\n","We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"05633bc3-9118-449b-9562-45fc437576c2","showTitle":false,"title":""}},"outputs":[],"source":["derived_features = [\n"," DerivedFeature(\n"," name=\"f_trip_time_distance\",\n"," feature_type=FLOAT,\n"," input_features=[\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," ],\n"," transform=\"f_trip_distance / f_trip_time_duration\",\n"," )\n","]"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ad102c45-586d-468c-85f0-9454401ef10b","showTitle":false,"title":""}},"source":["### Build features\n","\n","Finally, we build the features."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"91bb5ebb-87e4-470b-b8eb-1c89b351740e","showTitle":false,"title":""}},"outputs":[],"source":["client.build_features(\n"," anchor_list=[feature_anchor, agg_feature_anchor],\n"," derived_feature_list=derived_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"632d5f46-f9e2-41a8-aab7-34f75206e2aa","showTitle":false,"title":""}},"source":["## 3. Create Training Data Using Point-in-Time Correct Feature Join\n","\n","After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n","\n","To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n","what features and how these features should be joined to the observation data. \n","\n","To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"02feabc9-2f2f-43e8-898d-b28082798e98","showTitle":false,"title":""}},"outputs":[],"source":["feature_names = [feature.name for feature in features + agg_features + derived_features]\n","feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FORMAT = \"parquet\"\n","offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"67e81466-c736-47ba-b122-e640642c01cf","showTitle":false,"title":""}},"outputs":[],"source":["# Features that we want to request. Can use a subset of features\n","query = FeatureQuery(\n"," feature_list=feature_names,\n"," key=agg_key,\n",")\n","settings = ObservationSettings(\n"," observation_path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")\n","client.get_offline_features(\n"," observation_settings=settings,\n"," feature_query=query,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n"," execution_configurations=SparkExecutionConfiguration({\n"," \"spark.feathr.outputFormat\": DATA_FORMAT,\n"," }),\n"," output_path=offline_features_path,\n",")\n","\n","client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9871af55-25eb-41ee-a58a-fda74b1a174e","showTitle":false,"title":""}},"outputs":[],"source":["# Show feature results\n","df = get_result_df(\n"," spark=spark,\n"," client=client,\n"," data_format=\"parquet\",\n"," res_url=offline_features_path,\n",")\n","df.select(feature_names).limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f","showTitle":false,"title":""}},"source":["## 4. Train and Evaluate a Prediction Model\n","\n","After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n","\n","Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5a226026-1c7b-48db-8f91-88d5c2ddf023","showTitle":false,"title":""}},"source":["### Load Train and Test Data from the Offline Feature Values"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bd2cdc83-0920-46e8-9454-e5e6e7832ce0","showTitle":false,"title":""}},"outputs":[],"source":["# Train / test split\n","train_df, test_df = (\n"," df # Dataframe that we generated from get_offline_features call.\n"," .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n"," .where(F.col(\"f_trip_time_duration\") > 0)\n"," .fillna(0)\n"," .randomSplit([0.8, 0.2])\n",")\n","\n","print(f\"Num train samples: {train_df.count()}\")\n","print(f\"Num test samples: {test_df.count()}\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd","showTitle":false,"title":""}},"source":["### Build a ML Pipeline\n","\n","Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a254361-63e9-45b2-8c19-40549762eacb","showTitle":false,"title":""}},"outputs":[],"source":["# Generate a feature vector column for SparkML\n","vector_assembler = VectorAssembler(\n"," inputCols=[x for x in df.columns if x in feature_names],\n"," outputCol=\"features\",\n",")\n","\n","# Define a model\n","gbt = GBTRegressor(\n"," featuresCol=\"features\",\n"," maxIter=100,\n"," maxDepth=5,\n"," maxBins=16,\n",")\n","\n","# Create a ML pipeline\n","ml_pipeline = Pipeline(stages=[\n"," vector_assembler,\n"," gbt,\n","])"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bef93538-9591-4247-97b6-289d2055b7b1","showTitle":false,"title":""}},"source":["### Train and Evaluate the Model"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"0c3d5f35-11a3-4644-9992-5860169d8302","showTitle":false,"title":""}},"outputs":[],"source":["# Train a model\n","model = ml_pipeline.fit(train_df)\n","\n","# Make predictions\n","predictions = model.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1f9b584c-6228-4a02-a6c3-9b8dd2b78091","showTitle":false,"title":""}},"outputs":[],"source":["# Evaluate\n","evaluator = RegressionEvaluator(\n"," labelCol=\"label\",\n"," predictionCol=\"prediction\",\n",")\n","\n","rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n","mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n","print(f\"RMSE: {rmse}\\nMAE: {mae}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"25c33abd-6e87-437d-a6a1-86435f065a1e","showTitle":false,"title":""}},"outputs":[],"source":["# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n","predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n","\n","predictions_pdf.plot(\n"," x=\"index\",\n"," y=[\"label\", \"prediction\"],\n"," style=['-', ':'],\n"," figsize=(20, 10),\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"664d78cc-4a92-430c-9e05-565ba904558e","showTitle":false,"title":""}},"outputs":[],"source":["predictions_pdf.plot.scatter(\n"," x=\"label\",\n"," y=\"prediction\",\n"," xlim=(0, 100),\n"," ylim=(0, 100),\n"," figsize=(10, 10),\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8a56d165-c813-4ce0-8ae6-9f4d313c463d","showTitle":false,"title":""}},"source":["## 5. Materialize Feature Values for Online Scoring\n","\n","While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n","\n","Note, only the features anchored to offline data source can be materialized."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"751fa72e-8f94-40a1-994e-3e8315b51d37","showTitle":false,"title":""}},"outputs":[],"source":["materialized_feature_names = [feature.name for feature in agg_features]\n","materialized_feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4d4699ed-42e6-408f-903d-2f799284f4b6","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n","\n"," # Get the last date from the dataset\n"," backfill_timestamp = (\n"," df_raw\n"," .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n"," .agg({TIMESTAMP_COL: \"max\"})\n"," .collect()[0][0]\n"," )\n","\n"," # Time range to materialize\n"," backfill_time = BackfillTime(\n"," start=backfill_timestamp,\n"," end=backfill_timestamp,\n"," step=timedelta(days=1),\n"," )\n","\n"," # Destinations:\n"," # For online store,\n"," redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n","\n"," # For offline store,\n"," # adls_sink = HdfsSink(output_path=)\n","\n"," settings = MaterializationSettings(\n"," name=FEATURE_TABLE_NAME + \".job\", # job name\n"," backfill_time=backfill_time,\n"," sinks=[redis_sink], # or adls_sink\n"," feature_names=materialized_feature_names,\n"," )\n","\n"," client.materialize_features(\n"," settings=settings,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n"," )\n","\n"," client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9","showTitle":false,"title":""}},"source":["Now, you can retrieve features for online scoring as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"424bc9eb-a47f-4b46-be69-8218d55e66ad","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," # Note, to get a single key, you may use client.get_online_features instead\n"," materialized_feature_values = client.multi_get_online_features(\n"," feature_table=FEATURE_TABLE_NAME,\n"," keys=[\"239\", \"265\"],\n"," feature_names=materialized_feature_names,\n"," )\n"," materialized_feature_values"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3596dc71-a363-4b6a-a169-215c89978558","showTitle":false,"title":""}},"source":["## Cleanup"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"b5fb292e-bbb6-4dd7-8e79-c62d9533e820","showTitle":false,"title":""}},"outputs":[],"source":["# Remove temporary files\n","dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"databricks_quickstart_nyc_taxi_demo","notebookOrigID":2365994027381987,"widgets":{"REDIS_KEY":{"currentValue":"","nuid":"d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca","widgetInfo":{"defaultValue":"","label":null,"name":"REDIS_KEY","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}},"RESOURCE_PREFIX":{"currentValue":"","nuid":"87a26035-86fc-4dbd-8dd0-dc546c1c63c1","widgetInfo":{"defaultValue":"","label":null,"name":"RESOURCE_PREFIX","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}}}},"kernelspec":{"display_name":"Python 3.10.8 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.8"},"vscode":{"interpreter":{"hash":"b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"}}},"nbformat":4,"nbformat_minor":0}
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "843d3142-24ca-4bd1-9e31-b55163804fe3",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dbutils.widgets.text(\"RESOURCE_PREFIX\", \"\")\n",
+ "dbutils.widgets.text(\"REDIS_KEY\", \"\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "384e5e16-7213-4186-9d04-09d03b155534",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "# Feathr Feature Store on Databricks Demo Notebook\n",
+ "\n",
+ "This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).\n",
+ "\n",
+ "This notebook is specifically written for Databricks and is relying on some of the Databricks packages such as `dbutils`. The intention here is to provide a \"one click run\" example with minimum configuration. For example:\n",
+ "- This notebook skips feature registry which requires running Azure Purview. \n",
+ "- To make the online feature query work, you will need to configure the Redis endpoint. \n",
+ "\n",
+ "The full-fledged notebook can be found from [here](https://github.com/feathr-ai/feathr/blob/main/docs/samples/nyc_taxi_demo.ipynb)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "c2ce58c7-9263-469a-bbb7-43364ddb07b8",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## Prerequisite\n",
+ "\n",
+ "To use feathr materialization for online scoring with Redis cache, you may deploy a Redis cluster and set `RESOURCE_PREFIX` and `REDIS_KEY` via Databricks widgets. Note that the deployed Redis host address should be `{RESOURCE_PREFIX}redis.redis.cache.windows.net`. More details about how to deploy the Redis cluster can be found [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html#configurure-redis-cluster).\n",
+ "\n",
+ "To run this notebook, you'll need to install `feathr` pip package. Here, we install notebook-scoped library. For details, please see [Azure Databricks dependency management document](https://learn.microsoft.com/en-us/azure/databricks/libraries/)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "4609d7ad-ad74-40fc-b97e-f440a0fa0737",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Install feathr from the latest codes in the repo. You may use `pip install feathr` as well.\n",
+ "!pip install \"git+https://github.com/feathr-ai/feathr#subdirectory=feathr_project\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "c81fa80c-bca6-4ae5-84ad-659a036977bd",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## Notebook Steps\n",
+ "\n",
+ "This tutorial demonstrates the key capabilities of Feathr, including:\n",
+ "\n",
+ "1. Install Feathr and necessary dependencies.\n",
+ "1. Create shareable features with Feathr feature definition configs.\n",
+ "1. Create training data using point-in-time correct feature join\n",
+ "1. Train and evaluate a prediction model.\n",
+ "1. Materialize feature values for online scoring.\n",
+ "\n",
+ "The overall data flow is as follows:\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "80223a02-631c-40c8-91b3-a037249ffff9",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from datetime import datetime, timedelta\n",
+ "from math import sqrt\n",
+ "import os\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from pyspark.ml import Pipeline\n",
+ "from pyspark.ml.evaluation import RegressionEvaluator\n",
+ "from pyspark.ml.feature import VectorAssembler\n",
+ "from pyspark.ml.regression import GBTRegressor\n",
+ "from pyspark.sql import DataFrame\n",
+ "import pyspark.sql.functions as F\n",
+ "\n",
+ "import feathr\n",
+ "from feathr import (\n",
+ " FeathrClient,\n",
+ " # Feature data types\n",
+ " BOOLEAN,\n",
+ " FLOAT,\n",
+ " INT32,\n",
+ " ValueType,\n",
+ " # Feature data sources\n",
+ " INPUT_CONTEXT,\n",
+ " HdfsSource,\n",
+ " # Feature aggregations\n",
+ " TypedKey,\n",
+ " WindowAggTransformation,\n",
+ " # Feature types and anchor\n",
+ " DerivedFeature,\n",
+ " Feature,\n",
+ " FeatureAnchor,\n",
+ " # Materialization\n",
+ " BackfillTime,\n",
+ " MaterializationSettings,\n",
+ " RedisSink,\n",
+ " # Offline feature computation\n",
+ " FeatureQuery,\n",
+ " ObservationSettings,\n",
+ ")\n",
+ "from feathr.datasets import nyc_taxi\n",
+ "from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n",
+ "from feathr.utils.config import generate_config\n",
+ "from feathr.utils.job_utils import get_result_df\n",
+ "\n",
+ "\n",
+ "print(\n",
+ " f\"\"\"Feathr version: {feathr.__version__}\n",
+ "Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "ab35fa01-b392-457e-8fde-7e445a3c39b5",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## 2. Create Shareable Features with Feathr Feature Definition Configs\n",
+ "\n",
+ "In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n",
+ "Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "09f93a9f-7b33-4d91-8f31-ee3b20991696",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n",
+ "PROJECT_NAME = \"feathr_getting_started\"\n",
+ "\n",
+ "REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n",
+ "\n",
+ "# Use a databricks cluster\n",
+ "SPARK_CLUSTER = \"databricks\"\n",
+ "\n",
+ "# Databricks file system path\n",
+ "DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "331753d6-1850-47b5-ad97-84b7c01d79d1",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Redis credential\n",
+ "os.environ[\"REDIS_PASSWORD\"] = REDIS_KEY"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Configurations\n",
+ "\n",
+ "Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field.\n",
+ "\n",
+ "In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "8cd64e3a-376c-48e6-ba41-5197f3591d48",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "config_path = generate_config(\n",
+ " resource_prefix=RESOURCE_PREFIX,\n",
+ " project_name=PROJECT_NAME,\n",
+ " spark_cluster=SPARK_CLUSTER,\n",
+ " # You may set an existing cluster id here, but Databricks recommend to use new clusters for greater reliability.\n",
+ " cluster_name=None, # Set None to create a new job cluster\n",
+ ")\n",
+ "\n",
+ "with open(config_path, \"r\") as f:\n",
+ " print(f.read())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "58d22dc1-7590-494d-94ca-3e2488c31c8e",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "3fef7f2f-df19-4f53-90a5-ff7999ed983d",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Initialize Feathr Client"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "9713a2df-c7b2-4562-88b0-b7acce3cc43a",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "client = FeathrClient(config_path=config_path)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "c3b64bda-d42c-4a64-b976-0fb604cf38c5",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### View the NYC taxi fare dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "c4ccd7b3-298a-4e5a-8eec-b7e309db393e",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n",
+ "\n",
+ "# Download the data file\n",
+ "df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n",
+ "df_raw.limit(5).toPandas()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "7430c942-64e5-4b70-b823-16ce1d1b3cee",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Defining features with Feathr\n",
+ "\n",
+ "In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n",
+ "\n",
+ "* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n",
+ "* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n",
+ "* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n",
+ "\n",
+ "Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n",
+ "\n",
+ "There are two types of features -- anchored features and derivated features:\n",
+ "\n",
+ "* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n",
+ "* **Derived features**: Features that are computed on top of other features.\n",
+ "\n",
+ "#### Define anchored features\n",
+ "\n",
+ "A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "75b8d2ed-84df-4446-ae07-5f715434f3ea",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n",
+ "TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "93abbcc2-562b-47e4-ad4c-1fedd7cc64df",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# We define f_trip_distance and f_trip_time_duration features separately\n",
+ "# so that we can reuse them later for the derived features.\n",
+ "f_trip_distance = Feature(\n",
+ " name=\"f_trip_distance\",\n",
+ " feature_type=FLOAT,\n",
+ " transform=\"trip_distance\",\n",
+ ")\n",
+ "f_trip_time_duration = Feature(\n",
+ " name=\"f_trip_time_duration\",\n",
+ " feature_type=FLOAT,\n",
+ " transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",
+ ")\n",
+ "\n",
+ "features = [\n",
+ " f_trip_distance,\n",
+ " f_trip_time_duration,\n",
+ " Feature(\n",
+ " name=\"f_is_long_trip_distance\",\n",
+ " feature_type=BOOLEAN,\n",
+ " transform=\"trip_distance > 30.0\",\n",
+ " ),\n",
+ " Feature(\n",
+ " name=\"f_day_of_week\",\n",
+ " feature_type=INT32,\n",
+ " transform=\"dayofweek(lpep_dropoff_datetime)\",\n",
+ " ),\n",
+ " Feature(\n",
+ " name=\"f_day_of_month\",\n",
+ " feature_type=INT32,\n",
+ " transform=\"dayofmonth(lpep_dropoff_datetime)\",\n",
+ " ),\n",
+ " Feature(\n",
+ " name=\"f_hour_of_day\",\n",
+ " feature_type=INT32,\n",
+ " transform=\"hour(lpep_dropoff_datetime)\",\n",
+ " ),\n",
+ "]\n",
+ "\n",
+ "# After you have defined features, bring them together to build the anchor to the source.\n",
+ "feature_anchor = FeatureAnchor(\n",
+ " name=\"feature_anchor\",\n",
+ " source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n",
+ " features=features,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "728d2d5f-c11f-4941-bdc5-48507f5749f1",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "We can define the source with a preprocessing python function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "3cc59a0e-a41b-480e-a84e-ca5443d63143",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def preprocessing(df: DataFrame) -> DataFrame:\n",
+ " import pyspark.sql.functions as F\n",
+ "\n",
+ " df = df.withColumn(\n",
+ " \"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\")\n",
+ " )\n",
+ " return df\n",
+ "\n",
+ "\n",
+ "batch_source = HdfsSource(\n",
+ " name=\"nycTaxiBatchSource\",\n",
+ " path=DATA_FILE_PATH,\n",
+ " event_timestamp_column=TIMESTAMP_COL,\n",
+ " preprocessing=preprocessing,\n",
+ " timestamp_format=TIMESTAMP_FORMAT,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "46f863c4-bb81-434a-a448-6b585031a221",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "For the features with aggregation, the supported functions are as follows:\n",
+ "\n",
+ "| Aggregation Function | Input Type | Description |\n",
+ "| --- | --- | --- |\n",
+ "|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n",
+ "|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n",
+ "|LATEST| Any |Returns the latest not-null values from within the defined time window |"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "a373ecbe-a040-4cd3-9d87-0d5f4c5ba553",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "agg_key = TypedKey(\n",
+ " key_column=\"DOLocationID\",\n",
+ " key_column_type=ValueType.INT32,\n",
+ " description=\"location id in NYC\",\n",
+ " full_name=\"nyc_taxi.location_id\",\n",
+ ")\n",
+ "\n",
+ "agg_window = \"90d\"\n",
+ "\n",
+ "# Anchored features with aggregations\n",
+ "agg_features = [\n",
+ " Feature(\n",
+ " name=\"f_location_avg_fare\",\n",
+ " key=agg_key,\n",
+ " feature_type=FLOAT,\n",
+ " transform=WindowAggTransformation(\n",
+ " agg_expr=\"fare_amount_cents\",\n",
+ " agg_func=\"AVG\",\n",
+ " window=agg_window,\n",
+ " ),\n",
+ " ),\n",
+ " Feature(\n",
+ " name=\"f_location_max_fare\",\n",
+ " key=agg_key,\n",
+ " feature_type=FLOAT,\n",
+ " transform=WindowAggTransformation(\n",
+ " agg_expr=\"fare_amount_cents\",\n",
+ " agg_func=\"MAX\",\n",
+ " window=agg_window,\n",
+ " ),\n",
+ " ),\n",
+ "]\n",
+ "\n",
+ "agg_feature_anchor = FeatureAnchor(\n",
+ " name=\"agg_feature_anchor\",\n",
+ " source=batch_source, # External data source for feature. Typically a data table.\n",
+ " features=agg_features,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "149f85e2-fa3c-4895-b0c5-de5543ca9b6d",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "#### Define derived features\n",
+ "\n",
+ "We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "05633bc3-9118-449b-9562-45fc437576c2",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "derived_features = [\n",
+ " DerivedFeature(\n",
+ " name=\"f_trip_time_distance\",\n",
+ " feature_type=FLOAT,\n",
+ " input_features=[\n",
+ " f_trip_distance,\n",
+ " f_trip_time_duration,\n",
+ " ],\n",
+ " transform=\"f_trip_distance / f_trip_time_duration\",\n",
+ " )\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "ad102c45-586d-468c-85f0-9454401ef10b",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Build features\n",
+ "\n",
+ "Finally, we build the features."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "91bb5ebb-87e4-470b-b8eb-1c89b351740e",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "client.build_features(\n",
+ " anchor_list=[feature_anchor, agg_feature_anchor],\n",
+ " derived_feature_list=derived_features,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "632d5f46-f9e2-41a8-aab7-34f75206e2aa",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## 3. Create Training Data Using Point-in-Time Correct Feature Join\n",
+ "\n",
+ "After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n",
+ "\n",
+ "To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n",
+ "what features and how these features should be joined to the observation data. \n",
+ "\n",
+ "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "02feabc9-2f2f-43e8-898d-b28082798e98",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_names = [feature.name for feature in features + agg_features + derived_features]\n",
+ "feature_names"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "DATA_FORMAT = \"parquet\"\n",
+ "offline_features_path = str(\n",
+ " Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\")\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "67e81466-c736-47ba-b122-e640642c01cf",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Features that we want to request. Can use a subset of features\n",
+ "query = FeatureQuery(\n",
+ " feature_list=feature_names,\n",
+ " key=agg_key,\n",
+ ")\n",
+ "settings = ObservationSettings(\n",
+ " observation_path=DATA_FILE_PATH,\n",
+ " event_timestamp_column=TIMESTAMP_COL,\n",
+ " timestamp_format=TIMESTAMP_FORMAT,\n",
+ ")\n",
+ "client.get_offline_features(\n",
+ " observation_settings=settings,\n",
+ " feature_query=query,\n",
+ " # Note, execution_configurations argument only works when using a new job cluster\n",
+ " # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n",
+ " execution_configurations=SparkExecutionConfiguration(\n",
+ " {\n",
+ " \"spark.feathr.outputFormat\": DATA_FORMAT,\n",
+ " }\n",
+ " ),\n",
+ " output_path=offline_features_path,\n",
+ ")\n",
+ "\n",
+ "client.wait_job_to_finish(timeout_sec=500)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "9871af55-25eb-41ee-a58a-fda74b1a174e",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Show feature results\n",
+ "df = get_result_df(\n",
+ " spark=spark,\n",
+ " client=client,\n",
+ " data_format=\"parquet\",\n",
+ " res_url=offline_features_path,\n",
+ ")\n",
+ "df.select(feature_names).limit(5).toPandas()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## 4. Train and Evaluate a Prediction Model\n",
+ "\n",
+ "After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n",
+ "\n",
+ "Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "5a226026-1c7b-48db-8f91-88d5c2ddf023",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Load Train and Test Data from the Offline Feature Values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "bd2cdc83-0920-46e8-9454-e5e6e7832ce0",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Train / test split\n",
+ "train_df, test_df = (\n",
+ " df.withColumn( # Dataframe that we generated from get_offline_features call.\n",
+ " \"label\", F.col(\"fare_amount\").cast(\"double\")\n",
+ " )\n",
+ " .where(F.col(\"f_trip_time_duration\") > 0)\n",
+ " .fillna(0)\n",
+ " .randomSplit([0.8, 0.2])\n",
+ ")\n",
+ "\n",
+ "print(f\"Num train samples: {train_df.count()}\")\n",
+ "print(f\"Num test samples: {test_df.count()}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Build a ML Pipeline\n",
+ "\n",
+ "Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "2a254361-63e9-45b2-8c19-40549762eacb",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Generate a feature vector column for SparkML\n",
+ "vector_assembler = VectorAssembler(\n",
+ " inputCols=[x for x in df.columns if x in feature_names],\n",
+ " outputCol=\"features\",\n",
+ ")\n",
+ "\n",
+ "# Define a model\n",
+ "gbt = GBTRegressor(\n",
+ " featuresCol=\"features\",\n",
+ " maxIter=100,\n",
+ " maxDepth=5,\n",
+ " maxBins=16,\n",
+ ")\n",
+ "\n",
+ "# Create a ML pipeline\n",
+ "ml_pipeline = Pipeline(\n",
+ " stages=[\n",
+ " vector_assembler,\n",
+ " gbt,\n",
+ " ]\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "bef93538-9591-4247-97b6-289d2055b7b1",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "### Train and Evaluate the Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "0c3d5f35-11a3-4644-9992-5860169d8302",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Train a model\n",
+ "model = ml_pipeline.fit(train_df)\n",
+ "\n",
+ "# Make predictions\n",
+ "predictions = model.transform(test_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "1f9b584c-6228-4a02-a6c3-9b8dd2b78091",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Evaluate\n",
+ "evaluator = RegressionEvaluator(\n",
+ " labelCol=\"label\",\n",
+ " predictionCol=\"prediction\",\n",
+ ")\n",
+ "\n",
+ "rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n",
+ "mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n",
+ "print(f\"RMSE: {rmse}\\nMAE: {mae}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "25c33abd-6e87-437d-a6a1-86435f065a1e",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n",
+ "predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n",
+ "\n",
+ "predictions_pdf.plot(\n",
+ " x=\"index\",\n",
+ " y=[\"label\", \"prediction\"],\n",
+ " style=[\"-\", \":\"],\n",
+ " figsize=(20, 10),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "664d78cc-4a92-430c-9e05-565ba904558e",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "predictions_pdf.plot.scatter(\n",
+ " x=\"label\",\n",
+ " y=\"prediction\",\n",
+ " xlim=(0, 100),\n",
+ " ylim=(0, 100),\n",
+ " figsize=(10, 10),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "8a56d165-c813-4ce0-8ae6-9f4d313c463d",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## 5. Materialize Feature Values for Online Scoring\n",
+ "\n",
+ "While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n",
+ "\n",
+ "Note, only the features anchored to offline data source can be materialized."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "751fa72e-8f94-40a1-994e-3e8315b51d37",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "materialized_feature_names = [feature.name for feature in agg_features]\n",
+ "materialized_feature_names"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "4d4699ed-42e6-408f-903d-2f799284f4b6",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "if REDIS_KEY and RESOURCE_PREFIX:\n",
+ " FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n",
+ "\n",
+ " # Get the last date from the dataset\n",
+ " backfill_timestamp = (\n",
+ " df_raw.select(\n",
+ " F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL)\n",
+ " )\n",
+ " .agg({TIMESTAMP_COL: \"max\"})\n",
+ " .collect()[0][0]\n",
+ " )\n",
+ "\n",
+ " # Time range to materialize\n",
+ " backfill_time = BackfillTime(\n",
+ " start=backfill_timestamp,\n",
+ " end=backfill_timestamp,\n",
+ " step=timedelta(days=1),\n",
+ " )\n",
+ "\n",
+ " # Destinations:\n",
+ " # For online store,\n",
+ " redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n",
+ "\n",
+ " # For offline store,\n",
+ " # adls_sink = HdfsSink(output_path=)\n",
+ "\n",
+ " settings = MaterializationSettings(\n",
+ " name=FEATURE_TABLE_NAME + \".job\", # job name\n",
+ " backfill_time=backfill_time,\n",
+ " sinks=[redis_sink], # or adls_sink\n",
+ " feature_names=materialized_feature_names,\n",
+ " )\n",
+ "\n",
+ " client.materialize_features(\n",
+ " settings=settings,\n",
+ " # Note, execution_configurations argument only works when using a new job cluster\n",
+ " execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n",
+ " )\n",
+ "\n",
+ " client.wait_job_to_finish(timeout_sec=500)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "Now, you can retrieve features for online scoring as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "424bc9eb-a47f-4b46-be69-8218d55e66ad",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "if REDIS_KEY and RESOURCE_PREFIX:\n",
+ " # Note, to get a single key, you may use client.get_online_features instead\n",
+ " materialized_feature_values = client.multi_get_online_features(\n",
+ " feature_table=FEATURE_TABLE_NAME,\n",
+ " keys=[\"239\", \"265\"],\n",
+ " feature_names=materialized_feature_names,\n",
+ " )\n",
+ " materialized_feature_values"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "3596dc71-a363-4b6a-a169-215c89978558",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "source": [
+ "## Cleanup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "application/vnd.databricks.v1+cell": {
+ "inputWidgets": {},
+ "nuid": "b5fb292e-bbb6-4dd7-8e79-c62d9533e820",
+ "showTitle": false,
+ "title": ""
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Remove temporary files\n",
+ "dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"
+ ]
+ }
+ ],
+ "metadata": {
+ "application/vnd.databricks.v1+notebook": {
+ "dashboards": [],
+ "language": "python",
+ "notebookMetadata": {
+ "pythonIndentUnit": 4
+ },
+ "notebookName": "databricks_quickstart_nyc_taxi_demo",
+ "notebookOrigID": 2365994027381987,
+ "widgets": {
+ "REDIS_KEY": {
+ "currentValue": "",
+ "nuid": "d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca",
+ "widgetInfo": {
+ "defaultValue": "",
+ "label": null,
+ "name": "REDIS_KEY",
+ "options": {
+ "validationRegex": null,
+ "widgetType": "text"
+ },
+ "widgetType": "text"
+ }
+ },
+ "RESOURCE_PREFIX": {
+ "currentValue": "",
+ "nuid": "87a26035-86fc-4dbd-8dd0-dc546c1c63c1",
+ "widgetInfo": {
+ "defaultValue": "",
+ "label": null,
+ "name": "RESOURCE_PREFIX",
+ "options": {
+ "validationRegex": null,
+ "widgetType": "text"
+ },
+ "widgetType": "text"
+ }
+ }
+ }
+ },
+ "kernelspec": {
+ "display_name": "Python 3.10.4 ('feathr')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.4"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "e34a1a57d2e174682770a82d94a178aa36d3ccfaa21227c5d2308e319b7ae532"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/docs/samples/nyc_taxi_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb
index b789e9bf2..06b5cb340 100644
--- a/docs/samples/nyc_taxi_demo.ipynb
+++ b/docs/samples/nyc_taxi_demo.ipynb
@@ -89,18 +89,12 @@
},
"outputs": [],
"source": [
- "from datetime import datetime, timedelta\n",
- "import glob\n",
- "import json\n",
+ "from datetime import timedelta\n",
"from math import sqrt\n",
"import os\n",
"from pathlib import Path\n",
- "import requests\n",
"from tempfile import TemporaryDirectory\n",
"\n",
- "from azure.identity import AzureCliCredential, DefaultAzureCredential \n",
- "from azure.keyvault.secrets import SecretClient\n",
- "import pandas as pd\n",
"from pyspark.ml import Pipeline\n",
"from pyspark.ml.evaluation import RegressionEvaluator\n",
"from pyspark.ml.feature import VectorAssembler\n",
@@ -154,39 +148,44 @@
},
"outputs": [],
"source": [
- "RESOURCE_PREFIX = None # TODO fill the value\n",
+ "RESOURCE_PREFIX = None # TODO fill the value used to deploy the resources via ARM template\n",
"PROJECT_NAME = \"feathr_getting_started\"\n",
"\n",
- "# Data store root path. Could be a local file system path or Azure storage path like abfs or wasbs\n",
- "DATA_STORE_PATH = TemporaryDirectory().name\n",
- "\n",
"# Currently support: 'azure_synapse', 'databricks', and 'local' \n",
"SPARK_CLUSTER = \"local\"\n",
- "# TODO -- Synapse spark pool name or Databricks cluster id\n",
- "CLUSTER_NAME = None\n",
"\n",
- "# If set True, use an interactive browser authentication\n",
+ "# If \"azure_synapse\":\n",
+ "AZURE_SYNAPSE_SPARK_POOL = None # Set Synapse spark pool name to use an existing cluster\n",
+ "\n",
+ "# If \"databricks\":\n",
+ "DATABRICKS_CLUSTER_ID = None # Set Databricks cluster id to use an existing cluster\n",
+ "DATABRICKS_URL = None # Set Databricks workspace url to use databricks\n",
+ "\n",
+ "# Data store root path. Could be a local file system path, dbfs or Azure storage path like abfs or wasbs\n",
+ "DATA_STORE_PATH = TemporaryDirectory().name\n",
+ "\n",
+ "# If set True, use an interactive browser authentication to get the redis password.\n",
"USE_CLI_AUTH = False\n",
"\n",
+ "REGISTER_FEATURES = False\n",
+ "\n",
"# (For the notebook test pipeline) If true, use ScrapBook package to collect the results.\n",
"SCRAP_RESULTS = False"
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "KEY_VAULT = f\"{RESOURCE_PREFIX}kv\"\n",
- "KEY_VAULT_URI = f\"https://{KEY_VAULT}.vault.azure.net\"\n",
+ "To use Databricks as the feathr client's target platform, you may need to set a databricks token to an environment variable like:\n",
+ "\n",
+ "`export DATABRICKS_WORKSPACE_TOKEN_VALUE=your-token`\n",
+ "\n",
+ "or in the notebook cell,\n",
"\n",
- "ADLS_PATH = f\"abfss://{RESOURCE_PREFIX}fs@{RESOURCE_PREFIX}dls.dfs.core.windows.net/feathr_project\"\n",
+ "`os.environ[\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"] = your-token`\n",
"\n",
- "if SPARK_CLUSTER == \"azure_synapse\":\n",
- " os.environ['spark_config__azure_synapse__dev_url'] = f\"https://{resource_prefix}syws.dev.azuresynapse.net\"\n",
- " os.environ['spark_config__azure_synapse__pool_name'] = CLUSTER_NAME\n",
- " os.environ['spark_config__azure_synapse__workspace_dir'] = f\"abfss://{adls_fs_name}@{resource_prefix}dls.dfs.core.windows.net/{PROJECT_NAME}\""
+ "If you are running this notebook on Databricks, the token will be automatically retrieved by using the current Databricks notebook context."
]
},
{
@@ -195,22 +194,19 @@
"metadata": {},
"outputs": [],
"source": [
- "if USE_CLI_AUTH:\n",
- " !az login --use-device-code"
+ "# Force to use dbfs if the notebook is running on Databricks\n",
+ "if is_databricks() and not DATA_STORE_PATH.startswith(\"dbfs:\"):\n",
+ " DATA_STORE_PATH = f\"dbfs:/{DATA_STORE_PATH.lstrip('/')}\""
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
- "# Get all the required credentials from Azure Key Vault\n",
- "credential = AzureCliCredential() if USE_CLI_AUTH else DefaultAzureCredential()\n",
- "secret_client = SecretClient(vault_url=KEY_VAULT_URI, credential=credential)\n",
- "retrieved_secret = secret_client.get_secret('FEATHR-ONLINE-STORE-CONN').value"
+ "if USE_CLI_AUTH:\n",
+ " !az login --use-device-code"
]
},
{
@@ -219,27 +215,17 @@
"metadata": {},
"outputs": [],
"source": [
- "# Redis credential\n",
- "os.environ['REDIS_PASSWORD'] = retrieved_secret.split(\",\")[1].split(\"password=\", 1)[1]\n",
- "\n",
- "if SPARK_CLUSTER == \"local\":\n",
- " os.environ['SPARK_LOCAL_IP'] = \"127.0.0.1\"\n",
- "\n",
- "elif SPARK_CLUSTER == \"databricks\" and is_databricks():\n",
- " ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n",
- " databricks_config = {\n",
- " 'run_name': \"FEATHR_FILL_IN\",\n",
- " 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n",
- " 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n",
- " 'spark_jar_task': {\n",
- " 'main_class_name': \"FEATHR_FILL_IN\",\n",
- " 'parameters': [\"FEATHR_FILL_IN\"],\n",
- " },\n",
- " }\n",
- " os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n",
- " os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n",
- " os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n",
- " os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()\n"
+ "# Redis password\n",
+ "if 'REDIS_PASSWORD' not in os.environ:\n",
+ " # Try to get all the required credentials from Azure Key Vault\n",
+ " from azure.identity import AzureCliCredential, DefaultAzureCredential \n",
+ " from azure.keyvault.secrets import SecretClient\n",
+ "\n",
+ " vault_url = f\"https://{RESOURCE_PREFIX}kv.vault.azure.net\"\n",
+ " credential = AzureCliCredential() if USE_CLI_AUTH else DefaultAzureCredential()\n",
+ " secret_client = SecretClient(vault_url=vault_url, credential=credential)\n",
+ " retrieved_secret = secret_client.get_secret('FEATHR-ONLINE-STORE-CONN').value\n",
+ " os.environ['REDIS_PASSWORD'] = retrieved_secret.split(\",\")[1].split(\"password=\", 1)[1]\n"
]
},
{
@@ -271,7 +257,14 @@
},
"outputs": [],
"source": [
- "config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n",
+ "config_path = generate_config(\n",
+ " resource_prefix=RESOURCE_PREFIX,\n",
+ " project_name=PROJECT_NAME,\n",
+ " spark_cluster=SPARK_CLUSTER,\n",
+ " # cluster name will be ignored in \"local\" spark.\n",
+ " cluster_name=AZURE_SYNAPSE_SPARK_POOL if SPARK_CLUSTER == \"azure_synapse\" else DATABRICKS_CLUSTER_ID,\n",
+ " databricks_url=DATABRICKS_URL,\n",
+ ")\n",
"\n",
"with open(config_path, 'r') as f: \n",
" print(f.read())"
@@ -334,8 +327,8 @@
"metadata": {},
"outputs": [],
"source": [
- "# To run on a local spark, start a spark session:\n",
- "if SPARK_CLUSTER == \"local\":\n",
+ "# If the notebook is runnong on Jupyter, start a spark session:\n",
+ "if is_jupyter():\n",
" spark = (\n",
" SparkSession\n",
" .builder\n",
@@ -345,7 +338,7 @@
" .getOrCreate()\n",
" )\n",
" \n",
- "# Else, you must already have spark session object available in databricks or synapse."
+ "# Else, you must already have a spark session object available in databricks or synapse."
]
},
{
@@ -472,7 +465,23 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "We can define the source with a preprocessing python function."
+ "We can define the source with a preprocessing python function. In order to make the source data accessible from the target spark cluster, we upload the data file into either DBFS or Azure Blob Storage if needed."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Define data source path\n",
+ "if client.spark_runtime == \"local\" or (client.spark_runtime == \"databricks\" and is_databricks()):\n",
+ " # In local mode, we can use the same data path as the source.\n",
+ " # If the notebook is running on databricks, DATA_FILE_PATH should be already a dbfs path.\n",
+ " data_source_path = DATA_FILE_PATH\n",
+ "else:\n",
+ " # Otherwise, upload the local file to dbfs.\n",
+ " data_source_path = client.feathr_spark_launcher.upload_or_get_cloud_path(DATA_FILE_PATH) "
]
},
{
@@ -488,7 +497,7 @@
"\n",
"batch_source = HdfsSource(\n",
" name=\"nycTaxiBatchSource\",\n",
- " path=DATA_FILE_PATH,\n",
+ " path=data_source_path,\n",
" event_timestamp_column=TIMESTAMP_COL,\n",
" preprocessing=preprocessing,\n",
" timestamp_format=TIMESTAMP_FORMAT,\n",
@@ -692,7 +701,7 @@
" key=agg_key,\n",
")\n",
"settings = ObservationSettings(\n",
- " observation_path=DATA_FILE_PATH,\n",
+ " observation_path=data_source_path,\n",
" event_timestamp_column=TIMESTAMP_COL,\n",
" timestamp_format=TIMESTAMP_FORMAT,\n",
")\n",
@@ -889,12 +898,14 @@
"metadata": {},
"outputs": [],
"source": [
- "try:\n",
- " client.register_features()\n",
- "except KeyError:\n",
- " # TODO temporarily go around the \"Already exists\" error\n",
- " \n",
- " client.list_registered_features(project_name=PROJECT_NAME)"
+ "if REGISTER_FEATURES:\n",
+ " try:\n",
+ " client.register_features()\n",
+ " except KeyError:\n",
+ " # TODO temporarily go around the \"Already exists\" error\n",
+ " pass \n",
+ " print(client.list_registered_features(project_name=PROJECT_NAME))\n",
+ " # You can get the actual features too by calling client.get_features_from_registry(PROJECT_NAME)"
]
},
{
@@ -915,29 +926,6 @@
"Note, only the features anchored to offline data source can be materialized."
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Get registered features\n",
- "registered_features_dict = client.get_features_from_registry(PROJECT_NAME)\n",
- "\n",
- "observation_feature_names = []\n",
- "materialized_feature_names = []\n",
- "\n",
- "for feature_name, feature in registered_features_dict.items():\n",
- " if feature.key[0].key_column == \"NOT_NEEDED\":\n",
- " observation_feature_names.append(feature_name)\n",
- " else:\n",
- " materialized_feature_names.append(feature_name)\n",
- " \n",
- "print(f\"Features that will be extracted directly from the observation: {observation_feature_names}\")\n",
- "print(\"\")\n",
- "print(f\"Features that will be extracted from the source data and materialized to online storage: {materialized_feature_names}\")"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -988,7 +976,7 @@
" name=FEATURE_TABLE_NAME + \".job\", # job name\n",
" backfill_time=backfill_time,\n",
" sinks=[redis_sink], # or adls_sink\n",
- " feature_names=materialized_feature_names,\n",
+ " feature_names=[feature.name for feature in agg_features],\n",
")\n",
"\n",
"client.materialize_features(\n",
@@ -1016,7 +1004,7 @@
"materialized_feature_values = client.multi_get_online_features(\n",
" feature_table=FEATURE_TABLE_NAME,\n",
" keys=[\"239\", \"265\"],\n",
- " feature_names=materialized_feature_names,\n",
+ " feature_names=[feature.name for feature in agg_features],\n",
")\n",
"materialized_feature_values"
]
@@ -1034,7 +1022,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# TODO: Unregister or any other cleanups."
+ "# TODO: Unregister, delete cached files or do any other cleanups."
]
},
{
@@ -1083,7 +1071,7 @@
},
"celltoolbar": "Tags",
"kernelspec": {
- "display_name": "Python 3.10.4 ('feathr')",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1101,7 +1089,7 @@
},
"vscode": {
"interpreter": {
- "hash": "ddb0e38f168d5afaa0b8ab4851ddd8c14364f1d087c15de6ff2ee5a559aec1f2"
+ "hash": "e34a1a57d2e174682770a82d94a178aa36d3ccfaa21227c5d2308e319b7ae532"
}
}
},
diff --git a/feathr_project/feathr/datasets/nyc_taxi.py b/feathr_project/feathr/datasets/nyc_taxi.py
index ec605aae6..e00fa7150 100644
--- a/feathr_project/feathr/datasets/nyc_taxi.py
+++ b/feathr_project/feathr/datasets/nyc_taxi.py
@@ -73,7 +73,7 @@ def get_spark_df(
if is_databricks():
# Databricks uses "dbfs:/" prefix for spark paths
if not local_cache_path.startswith("dbfs:"):
- local_cache_path = str(Path("dbfs:", local_cache_path.lstrip("/")))
+ local_cache_path = f"dbfs:/{local_cache_path.lstrip('/')}"
# Databricks uses "/dbfs/" prefix for python paths
python_local_cache_path = local_cache_path.replace("dbfs:", "/dbfs")
# TODO add "if is_synapse()"
diff --git a/feathr_project/feathr/utils/config.py b/feathr_project/feathr/utils/config.py
index 9a9438567..47ac84679 100644
--- a/feathr_project/feathr/utils/config.py
+++ b/feathr_project/feathr/utils/config.py
@@ -1,61 +1,209 @@
+from copy import deepcopy
+import os
+import json
from tempfile import NamedTemporaryFile
+from typing import Dict
+import yaml
+from feathr.utils.platform import is_databricks
-FEATHR_CONFIG_TEMPLATE = """
-api_version: 1
-project_config:
- project_name: {project_name}
+DEFAULT_FEATHR_CONFIG = {
+ "api_version": 1,
+ "project_config": {}, # "project_name"
+ "feature_registry": {}, # "api_endpoint"
+ "spark_config": {
+ # "spark_cluster". Currently support 'azure_synapse', 'databricks', and 'local'
+ "spark_result_output_parts": "1",
+ },
+ "offline_store": {
+ "adls": {"adls_enabled": "true"},
+ "wasb": {"wasb_enabled": "true"},
+ },
+ "online_store": {
+ "redis": {
+ # "host"
+ "port": "6380",
+ "ssl_enabled": "true",
+ }
+ }
+}
-feature_registry:
- api_endpoint: 'https://{resource_prefix}webapp.azurewebsites.net/api/v1'
-spark_config:
- # Currently support: 'azure_synapse', 'databricks', and 'local'
- spark_cluster: {spark_cluster}
- spark_result_output_parts: '1'
+# New databricks job cluster config
+DEFAULT_DATABRICKS_CLUSTER_CONFIG = {
+ "spark_version": "11.2.x-scala2.12",
+ "node_type_id": "Standard_D3_v2",
+ "num_workers": 2,
+ "spark_conf": {
+ "FEATHR_FILL_IN": "FEATHR_FILL_IN",
+ # Exclude conflicting packages if use feathr <= v0.8.0:
+ "spark.jars.excludes": "commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api",
+ },
+}
-offline_store:
- wasb:
- wasb_enabled: true
-online_store:
- # You can skip this part if you don't have Redis and skip materialization later in this notebook.
- redis:
- host: '{resource_prefix}redis.redis.cache.windows.net'
- port: 6380
- ssl_enabled: true
-"""
+# New Azure Synapse spark pool config
+DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG = {
+ "executor_size": "Small",
+ "executor_num": 2,
+}
def generate_config(
resource_prefix: str,
project_name: str,
spark_cluster: str,
+ cluster_name: str = None,
+ databricks_url: str = None,
output_filepath: str = None,
+ use_env_vars: bool = True,
) -> str:
- """Generate a feathr config yaml file
+ """Generate a feathr config yaml file. Note, if environment variables are set, they will be used instead of the
+ provided arguments.
+
+ Some credential variables are intentionally not included in the argument and the outut config file
+ to avoid leaking secrets. E.g. DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD.
+ Those values should be passed via the environment variables regardless of the `use_env_vars` flag.
+
+ Note:
+ This utility function assumes Azure resources are deployed using the Azure Resource Manager (ARM) template,
+ and infers resource names based on the given `resource_prefix`. If you deploy resources manually, you may need
+ to create the config file manually.
Args:
resource_prefix: Resource name prefix.
project_name: Project name.
spark_cluster: Spark cluster to use. Either 'local', 'databricks', or 'azure_synapse'.
- output_filepath: Output filepath.
+ cluster_name (optional): Synapse spark pool name or Databricks cluster id if applicable.
+ If not provided, a new (job) cluster will be created and used.
+ databricks_url (optional): Databricks workspace url if applicable.
+ output_filepath (optional): Output filepath.
+ use_env_vars (optional): Whether to use environment variables if they are set.
Returns:
str: Generated config file path. output_filepath if provided. Otherwise, NamedTemporaryFile path.
"""
+ if use_env_vars:
+ spark_cluster = os.getenv("SPARK_CONFIG__SPARK_CLUSTER", spark_cluster)
- conf_str = FEATHR_CONFIG_TEMPLATE.format(
- resource_prefix=resource_prefix,
- project_name=project_name,
- spark_cluster=spark_cluster,
- )
+ config = deepcopy(DEFAULT_FEATHR_CONFIG)
+ config["project_config"]["project_name"] = project_name
+ config["feature_registry"]["api_endpoint"] = f"https://{resource_prefix}webapp.azurewebsites.net/api/v1"
+ config["spark_config"]["spark_cluster"] = spark_cluster
+ config["online_store"]["redis"]["host"] = f"{resource_prefix}redis.redis.cache.windows.net"
+
+ # Set platform specific configurations
+ if spark_cluster == "local":
+ _set_local_spark_config()
+ elif spark_cluster == "azure_synapse":
+ _set_azure_synapse_config(
+ config=config,
+ resource_prefix=resource_prefix,
+ project_name=project_name,
+ cluster_name=cluster_name,
+ use_env_vars=use_env_vars,
+ )
+ elif spark_cluster == "databricks":
+ _set_databricks_config(
+ config=config,
+ project_name=project_name,
+ workspace_url=databricks_url,
+ cluster_name=cluster_name,
+ use_env_vars=use_env_vars,
+ )
if not output_filepath:
output_filepath = NamedTemporaryFile(mode="w", delete=False).name
- with open(output_filepath, "w") as conf_file:
- conf_file.write(conf_str)
+ with open(output_filepath, "w") as f:
+ yaml.dump(config, f, default_flow_style=False)
return output_filepath
+
+
+def _set_local_spark_config():
+ """Set environment variables for local spark cluster."""
+ os.environ["SPARK_LOCAL_IP"] = os.getenv(
+ "SPARK_LOCAL_IP",
+ "127.0.0.1",
+ )
+
+
+def _set_azure_synapse_config(
+ config: Dict,
+ resource_prefix: str,
+ project_name: str,
+ cluster_name: str = None,
+ use_env_vars: bool = True,
+):
+ """Set environment variables for Azure Synapse spark cluster.
+ One may need to set ADLS_KEY"""
+
+ dev_url = f"https://{resource_prefix}syws.dev.azuresynapse.net"
+ workspace_dir = f"abfss://{resource_prefix}fs@{resource_prefix}dls.dfs.core.windows.net/{project_name}"
+
+ if use_env_vars:
+ dev_url = os.getenv("SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL", dev_url)
+ cluster_name = os.getenv("SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME", cluster_name)
+ workspace_dir = os.getenv("SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR", workspace_dir)
+
+ if not cluster_name:
+ raise ValueError("Azure Synapse spark pool name is not provided.")
+
+ config["spark_config"]["azure_synapse"] = {
+ "dev_url": dev_url,
+ "pool_name": cluster_name,
+ "workspace_dir": workspace_dir,
+ **DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG,
+ }
+
+
+def _set_databricks_config(
+ config: Dict,
+ project_name: str,
+ workspace_url: str,
+ cluster_name: str = None,
+ use_env_vars: bool = True,
+):
+ if is_databricks():
+ # If this functions is being called in Databricks, we may use the context to override the provided arguments.
+ ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()
+ workspace_url = "https://" + ctx.tags().get("browserHostName").get()
+ workspace_token = ctx.apiToken().get()
+ else:
+ workspace_token = os.getenv("DATABRICKS_WORKSPACE_TOKEN_VALUE", None)
+
+ work_dir = f"dbfs:/{project_name}"
+ databricks_config = {
+ "run_name": "FEATHR_FILL_IN",
+ "libraries": [{"jar": "FEATHR_FILL_IN"}],
+ "spark_jar_task": {
+ "main_class_name": "FEATHR_FILL_IN",
+ "parameters": ["FEATHR_FILL_IN"],
+ },
+ }
+ if cluster_name is None:
+ databricks_config["new_cluster"] = DEFAULT_DATABRICKS_CLUSTER_CONFIG
+ else:
+ databricks_config["existing_cluster_id"] = cluster_name
+ config_template = json.dumps(databricks_config)
+
+ if use_env_vars:
+ work_dir = os.getenv("SPARK_CONFIG__DATABRICKS__WORK_DIR", work_dir)
+ workspace_url = os.getenv("SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL", workspace_url)
+ workspace_token = os.getenv("DATABRICKS_WORKSPACE_TOKEN_VALUE", workspace_token)
+ config_template = os.getenv("SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE", config_template)
+
+ if not workspace_url:
+ raise ValueError("Databricks workspace url is not provided.")
+
+ if not workspace_token:
+ raise ValueError("Databricks workspace token is not provided.")
+
+ os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = workspace_token
+ config["spark_config"]["databricks"] = {
+ "work_dir": work_dir,
+ "workspace_instance_url": workspace_url,
+ "config_template": config_template,
+ }
diff --git a/feathr_project/feathr/utils/job_utils.py b/feathr_project/feathr/utils/job_utils.py
index 12f27c2cb..6f5814e43 100644
--- a/feathr_project/feathr/utils/job_utils.py
+++ b/feathr_project/feathr/utils/job_utils.py
@@ -101,9 +101,10 @@ def get_result_df(
elif client.spark_runtime == "databricks":
if not res_url.startswith("dbfs:"):
- raise ValueError(
- f"In Databricks, the result files are expected to be stored at a DBFS storage but res_url = {res_url}."
+ logger.warning(
+ f"In Databricks, the result files are expected to be stored in DBFS, but the res_url {res_url} is not a dbfs path. Prefixing it with 'dbfs:/'"
)
+ res_url = f"dbfs:/{res_url.lstrip('/')}"
if is_databricks(): # Check if the function is being called from Databricks
if local_cache_path is not None:
@@ -111,12 +112,9 @@ def get_result_df(
"Result files are already in DBFS and thus `local_cache_path` will be ignored."
)
local_cache_path = res_url
- elif local_cache_path is None: # Download the result from dbfs to local
- local_cache_path = TemporaryDirectory().name
- else:
- logger.warning("This utility function currently supports local spark and databricks. You may encounter unexpected results on other platforms.")
- # TODO elif azure_synapse
+ if local_cache_path is None:
+ local_cache_path = TemporaryDirectory().name
if local_cache_path != res_url:
logger.info(f"{res_url} files will be downloaded into {local_cache_path}")
diff --git a/feathr_project/pyproject.toml b/feathr_project/pyproject.toml
index 5b7b2fc11..338a0eed3 100644
--- a/feathr_project/pyproject.toml
+++ b/feathr_project/pyproject.toml
@@ -11,7 +11,7 @@ multi_line_output = 3
[tool.pytest.ini_options]
markers = [
- "notebooks: tests Jupyter notebooks",
+ "notebooks: tests Jupyter notebooks"
]
[build-system]
diff --git a/feathr_project/test/conftest.py b/feathr_project/test/conftest.py
index b8ee3f345..52b10cf89 100644
--- a/feathr_project/test/conftest.py
+++ b/feathr_project/test/conftest.py
@@ -5,6 +5,21 @@
from feathr import FeathrClient
+def pytest_addoption(parser):
+ """Pytest command line argument options.
+ E.g.
+ `python -m pytest feathr_project/test/ --resource-prefix your_feathr_resource_prefix`
+ """
+ parser.addoption(
+ "--resource-prefix", action="store", default="feathrazuretest3", help="Test Azure resource prefix"
+ )
+
+
+@pytest.fixture
+def resource_prefix(request):
+ return request.config.getoption("--resource-prefix")
+
+
@pytest.fixture(scope="session")
def workspace_dir() -> str:
"""Workspace directory path containing data files and configs for testing."""
diff --git a/feathr_project/test/samples/test_notebooks.py b/feathr_project/test/samples/test_notebooks.py
index 778b157d7..f87cbff2e 100644
--- a/feathr_project/test/samples/test_notebooks.py
+++ b/feathr_project/test/samples/test_notebooks.py
@@ -23,22 +23,25 @@
@pytest.mark.notebooks
-def test__nyc_taxi_demo(tmp_path):
+def test__nyc_taxi_demo(resource_prefix, tmp_path):
notebook_name = "nyc_taxi_demo"
output_tmpdir = TemporaryDirectory()
output_notebook_path = str(tmp_path.joinpath(f"{notebook_name}.ipynb"))
+ print(f"Running {notebook_name} notebook as {output_notebook_path}")
+
pm.execute_notebook(
input_path=NOTEBOOK_PATHS[notebook_name],
output_path=output_notebook_path,
# kernel_name="python3",
parameters=dict(
- RESOURCE_PREFIX="feathrazuretest3", # Use the test resource group
+ RESOURCE_PREFIX=resource_prefix,
PROJECT_NAME=notebook_name,
DATA_STORE_PATH=output_tmpdir.name,
SPARK_CLUSTER="local",
USE_CLI_AUTH=False,
+ REGISTER_FEATURES=False,
SCRAP_RESULTS=True,
),
)
@@ -47,10 +50,7 @@ def test__nyc_taxi_demo(tmp_path):
nb = sb.read_notebook(output_notebook_path)
outputs = nb.scraps
- assert outputs["materialized_feature_values"].data["239"] == pytest.approx([5707., 1480.], abs=1.)
- assert outputs["materialized_feature_values"].data["265"] == pytest.approx([10000., 4160.], abs=1.)
+ assert outputs["materialized_feature_values"].data["239"] == pytest.approx([1480., 5707.], abs=1.)
+ assert outputs["materialized_feature_values"].data["265"] == pytest.approx([4160., 10000.], abs=1.)
assert outputs["rmse"].data == pytest.approx(5., abs=2.)
assert outputs["mae"].data == pytest.approx(2., abs=1.)
-
- # clean up
- output_tmpdir.cleanup()
diff --git a/feathr_project/test/unit/utils/test_config.py b/feathr_project/test/unit/utils/test_config.py
index 52adcae39..7b3395bc9 100644
--- a/feathr_project/test/unit/utils/test_config.py
+++ b/feathr_project/test/unit/utils/test_config.py
@@ -1,36 +1,38 @@
+from copy import deepcopy
+import os
from pathlib import Path
+import yaml
import pytest
-from feathr.utils.config import FEATHR_CONFIG_TEMPLATE, generate_config
-
-
-@pytest.fixture(scope="session")
-def feathr_config_str() -> str:
- return FEATHR_CONFIG_TEMPLATE.format(
- resource_prefix="test_prefix",
- project_name="test_project",
- spark_cluster="local",
- )
+from feathr import FeathrClient
+from feathr.utils.config import generate_config
@pytest.mark.parametrize(
"output_filepath", [None, "config.yml"],
)
-def test__generate_config(
+def test__generate_config__output_filepath(
output_filepath: str,
- feathr_config_str: str,
tmp_path: Path,
):
+ resource_prefix = "test_prefix"
+ project_name = "test_project"
+ spark_cluster = "local"
+
# Use tmp_path so that the test files get cleaned up after the tests
if output_filepath:
output_filepath = str(tmp_path / output_filepath)
+ if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
+ os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = "test_token"
+
config_filepath = generate_config(
- resource_prefix="test_prefix",
- project_name="test_project",
- spark_cluster="local",
+ resource_prefix=resource_prefix,
+ project_name=project_name,
+ spark_cluster=spark_cluster,
output_filepath=output_filepath,
+ use_env_vars=False,
)
# Assert if the config file was generated in the specified output path.
@@ -39,4 +41,67 @@ def test__generate_config(
# Assert the generated config string is correct.
with open(config_filepath, "r") as f:
- assert feathr_config_str == f.read()
+ config = yaml.safe_load(f)
+
+ assert config["project_config"]["project_name"] == project_name
+ assert config["feature_registry"]["api_endpoint"] == f"https://{resource_prefix}webapp.azurewebsites.net/api/v1"
+ assert config["spark_config"]["spark_cluster"] == spark_cluster
+ assert config["online_store"]["redis"]["host"] == f"{resource_prefix}redis.redis.cache.windows.net"
+
+
+@pytest.mark.parametrize(
+ "spark_cluster,cluster_name,databricks_url",
+ [
+ ("local", None, None),
+ ("databricks", None, "https://test_url"),
+ ("azure_synapse", "some_spark_pool", None),
+ ]
+)
+def test__generate_config__spark_cluster(
+ spark_cluster: str,
+ cluster_name: str,
+ databricks_url: str,
+):
+ """Test if spark cluster specific configs are generated without errors.
+ TODO - For now, this test doesn't check if the config values are correct.
+ """
+
+ if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
+ os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = "test_token"
+
+ generate_config(
+ resource_prefix="test_prefix",
+ project_name="test_project",
+ spark_cluster=spark_cluster,
+ cluster_name=cluster_name,
+ databricks_url=databricks_url,
+ use_env_vars=False,
+ )
+
+
+@pytest.mark.parametrize(
+ "spark_cluster,cluster_name,databricks_url",
+ [
+ ("databricks", "some_cluster_id", None),
+ ("azure_synapse", None, "https://test_url"),
+ ]
+)
+def test__generate_config__exceptions(
+ spark_cluster: str,
+ cluster_name: str,
+ databricks_url: str,
+):
+ """Test if exceptions are raised when databricks url and token are not provided."""
+
+ if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
+ os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = "test_token"
+
+ with pytest.raises(ValueError):
+ generate_config(
+ resource_prefix="test_prefix",
+ project_name="test_project",
+ spark_cluster=spark_cluster,
+ cluster_name=cluster_name,
+ databricks_url=databricks_url,
+ use_env_vars=False,
+ )
diff --git a/feathr_project/test/unit/utils/test_job_utils.py b/feathr_project/test/unit/utils/test_job_utils.py
index 1e005855e..136bc3545 100644
--- a/feathr_project/test/unit/utils/test_job_utils.py
+++ b/feathr_project/test/unit/utils/test_job_utils.py
@@ -91,10 +91,6 @@ def test__get_result_df__with_local_cache_path(
# Test ValueError when res_url is None
(False, "local", None, ValueError),
(True, "databricks", None, ValueError),
- # Test ValueError when res_url is not a dbfs path but client.spark_runtime is databricks
- (False, "databricks", "some_local_path", ValueError),
- # Test ValueError when res_url does not exists or not able to access.
- (False, "local", "some_doesnt_exist_path", Exception),
]
)
def test__get_result_df__exceptions(
@@ -113,6 +109,9 @@ def test__get_result_df__exceptions(
# Mock is_data_bricks
mocker.patch("feathr.utils.job_utils.is_databricks", return_value=is_databricks)
+ # Mock _load_files_to_pandas_df
+ mocker.patch("feathr.utils.job_utils._load_files_to_pandas_df")
+
with pytest.raises(expected_error):
get_result_df(client)
From 1f3894a0db20f4240d9ff40630b8cea4a83563a2 Mon Sep 17 00:00:00 2001
From: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
Date: Tue, 8 Nov 2022 04:11:53 +0000
Subject: [PATCH 14/18] Fix generate_config to accept all the feathr env var
config name
Signed-off-by: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
---
.../databricks_quickstart_nyc_taxi_demo.ipynb | 15 +-
docs/samples/nyc_taxi_demo.ipynb | 8 +-
feathr_project/feathr/utils/config.py | 241 +++++++++++-------
feathr_project/test/unit/utils/test_config.py | 42 ++-
4 files changed, 181 insertions(+), 125 deletions(-)
diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb
index 4dc58eaf7..e562ec5db 100644
--- a/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb
+++ b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb
@@ -240,6 +240,15 @@
"In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -256,9 +265,11 @@
"config_path = generate_config(\n",
" resource_prefix=RESOURCE_PREFIX,\n",
" project_name=PROJECT_NAME,\n",
- " spark_cluster=SPARK_CLUSTER,\n",
+ " spark_config__spark_cluster=SPARK_CLUSTER,\n",
" # You may set an existing cluster id here, but Databricks recommend to use new clusters for greater reliability.\n",
- " cluster_name=None, # Set None to create a new job cluster\n",
+ " databricks_cluster_id=None, # Set None to create a new job cluster\n",
+ " databricks_workspace_token_value=ctx.apiToken().get(),\n",
+ " spark_config__databricks__workspace_instance_url=f\"https://{ctx.tags().get('browserHostName').get()}\",\n",
")\n",
"\n",
"with open(config_path, \"r\") as f:\n",
diff --git a/docs/samples/nyc_taxi_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb
index 06b5cb340..10e189251 100644
--- a/docs/samples/nyc_taxi_demo.ipynb
+++ b/docs/samples/nyc_taxi_demo.ipynb
@@ -154,9 +154,6 @@
"# Currently support: 'azure_synapse', 'databricks', and 'local' \n",
"SPARK_CLUSTER = \"local\"\n",
"\n",
- "# If \"azure_synapse\":\n",
- "AZURE_SYNAPSE_SPARK_POOL = None # Set Synapse spark pool name to use an existing cluster\n",
- "\n",
"# If \"databricks\":\n",
"DATABRICKS_CLUSTER_ID = None # Set Databricks cluster id to use an existing cluster\n",
"DATABRICKS_URL = None # Set Databricks workspace url to use databricks\n",
@@ -261,9 +258,8 @@
" resource_prefix=RESOURCE_PREFIX,\n",
" project_name=PROJECT_NAME,\n",
" spark_cluster=SPARK_CLUSTER,\n",
- " # cluster name will be ignored in \"local\" spark.\n",
- " cluster_name=AZURE_SYNAPSE_SPARK_POOL if SPARK_CLUSTER == \"azure_synapse\" else DATABRICKS_CLUSTER_ID,\n",
- " databricks_url=DATABRICKS_URL,\n",
+ " databricks_cluster_id=DATABRICKS_CLUSTER_ID if SPARK_CLUSTER == \"databricks\" else None,\n",
+ " spark_config__databricks__workspace_instance_url=DATABRICKS_URL,\n",
")\n",
"\n",
"with open(config_path, 'r') as f: \n",
diff --git a/feathr_project/feathr/utils/config.py b/feathr_project/feathr/utils/config.py
index 47ac84679..27b41e8c7 100644
--- a/feathr_project/feathr/utils/config.py
+++ b/feathr_project/feathr/utils/config.py
@@ -1,3 +1,4 @@
+import collections.abc
from copy import deepcopy
import os
import json
@@ -13,7 +14,7 @@
"project_config": {}, # "project_name"
"feature_registry": {}, # "api_endpoint"
"spark_config": {
- # "spark_cluster". Currently support 'azure_synapse', 'databricks', and 'local'
+ "spark_cluster": "local", # Currently support 'azure_synapse', 'databricks', and 'local'
"spark_result_output_parts": "1",
},
"offline_store": {
@@ -47,72 +48,97 @@
DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG = {
"executor_size": "Small",
"executor_num": 2,
+ "pool_name": "spark3",
}
def generate_config(
resource_prefix: str,
project_name: str,
- spark_cluster: str,
- cluster_name: str = None,
- databricks_url: str = None,
output_filepath: str = None,
+ databricks_workspace_token_value: str = None,
+ databricks_cluster_id: str = None,
+ redis_password: str = None,
+ adls_key: str = None,
use_env_vars: bool = True,
+ **kwargs,
) -> str:
- """Generate a feathr config yaml file. Note, if environment variables are set, they will be used instead of the
- provided arguments.
+ """Generate a feathr config yaml file.
+ Note, `use_env_vars` argument gives an option to either use environment variables for generating the config file
+ or not. Feathr client will use environment variables anyway if they are set.
- Some credential variables are intentionally not included in the argument and the outut config file
- to avoid leaking secrets. E.g. DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD.
- Those values should be passed via the environment variables regardless of the `use_env_vars` flag.
+ Keyword arguments follow the same naming convention as the feathr config. E.g. to set Databricks as the target
+ cluster, use `spark_config__spark_cluster="databricks"`.
+ See https://feathr-ai.github.io/feathr/quickstart_synapse.html#step-4-update-feathr-config for more details.
Note:
This utility function assumes Azure resources are deployed using the Azure Resource Manager (ARM) template,
and infers resource names based on the given `resource_prefix`. If you deploy resources manually, you may need
- to create the config file manually.
+ to pass each resource url manually, e.g. `spark_config__azure_synapse__dev_url="your-resource-url"`.
Args:
- resource_prefix: Resource name prefix.
- project_name: Project name.
- spark_cluster: Spark cluster to use. Either 'local', 'databricks', or 'azure_synapse'.
- cluster_name (optional): Synapse spark pool name or Databricks cluster id if applicable.
- If not provided, a new (job) cluster will be created and used.
- databricks_url (optional): Databricks workspace url if applicable.
+ resource_prefix: Resource name prefix used when deploying Feathr resources by using ARM template.
+ project_name: Feathr project name.
+ cluster_name (optional): Databricks cluster or Azure Synapse spark pool name to use an existing one.
output_filepath (optional): Output filepath.
use_env_vars (optional): Whether to use environment variables if they are set.
+ databricks_workspace_token_value (optional): Databricks workspace token. If provided, the value will be stored
+ as the environment variable.
+ databricks_cluster_id (optional): Databricks cluster id to use an existing cluster.
+ redis_password (optional): Redis password. If provided, the value will be stored as the environment variable.
+ adls_key (optional): ADLS key. If provided, the value will be stored as the environment variable.
Returns:
- str: Generated config file path. output_filepath if provided. Otherwise, NamedTemporaryFile path.
+ str: Generated config file path. This will be identical to `output_filepath` if provided.
"""
- if use_env_vars:
- spark_cluster = os.getenv("SPARK_CONFIG__SPARK_CLUSTER", spark_cluster)
-
+ # Set keys
+ if databricks_workspace_token_value:
+ os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = databricks_workspace_token_value
+ if redis_password:
+ os.environ["REDIS_PASSWORD"] = redis_password
+ if adls_key:
+ os.environ["ADLS_KEY"] = adls_key
+
+ # Set configs
config = deepcopy(DEFAULT_FEATHR_CONFIG)
config["project_config"]["project_name"] = project_name
config["feature_registry"]["api_endpoint"] = f"https://{resource_prefix}webapp.azurewebsites.net/api/v1"
- config["spark_config"]["spark_cluster"] = spark_cluster
config["online_store"]["redis"]["host"] = f"{resource_prefix}redis.redis.cache.windows.net"
+ # Update configs using kwargs
+ new_config = _config_kwargs_to_dict(**kwargs)
+ _update_config(config, new_config)
+
# Set platform specific configurations
- if spark_cluster == "local":
+ if config["spark_config"]["spark_cluster"] == "local":
_set_local_spark_config()
- elif spark_cluster == "azure_synapse":
+ elif config["spark_config"]["spark_cluster"] == "azure_synapse":
_set_azure_synapse_config(
config=config,
resource_prefix=resource_prefix,
project_name=project_name,
- cluster_name=cluster_name,
- use_env_vars=use_env_vars,
)
- elif spark_cluster == "databricks":
+ elif config["spark_config"]["spark_cluster"] == "databricks":
_set_databricks_config(
config=config,
project_name=project_name,
- workspace_url=databricks_url,
- cluster_name=cluster_name,
- use_env_vars=use_env_vars,
+ cluster_id=databricks_cluster_id,
)
+ # Maybe update configs with environment variables
+ if use_env_vars:
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__SPARK_CLUSTER")
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL")
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME")
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR")
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__DATABRICKS__WORK_DIR")
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL")
+ _maybe_update_config_with_env_var(config, "SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE")
+
+ # Verify config
+ _verify_config(config)
+
+ # Write config to file
if not output_filepath:
output_filepath = NamedTemporaryFile(mode="w", delete=False).name
@@ -134,76 +160,107 @@ def _set_azure_synapse_config(
config: Dict,
resource_prefix: str,
project_name: str,
- cluster_name: str = None,
- use_env_vars: bool = True,
):
- """Set environment variables for Azure Synapse spark cluster.
- One may need to set ADLS_KEY"""
+ """Set environment variables for Azure Synapse spark cluster."""
- dev_url = f"https://{resource_prefix}syws.dev.azuresynapse.net"
- workspace_dir = f"abfss://{resource_prefix}fs@{resource_prefix}dls.dfs.core.windows.net/{project_name}"
+ if "azure_synapse" not in config["spark_config"]:
+ config["spark_config"]["azure_synapse"] = dict()
- if use_env_vars:
- dev_url = os.getenv("SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL", dev_url)
- cluster_name = os.getenv("SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME", cluster_name)
- workspace_dir = os.getenv("SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR", workspace_dir)
-
- if not cluster_name:
- raise ValueError("Azure Synapse spark pool name is not provided.")
-
- config["spark_config"]["azure_synapse"] = {
- "dev_url": dev_url,
- "pool_name": cluster_name,
- "workspace_dir": workspace_dir,
- **DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG,
- }
+ if "dev_url" not in config["spark_config"]["azure_synapse"]:
+ config["spark_config"]["azure_synapse"]["dev_url"] = f"https://{resource_prefix}syws.dev.azuresynapse.net"
+
+ if "workspace_dir" not in config["spark_config"]["azure_synapse"]:
+ config["spark_config"]["azure_synapse"]["workspace_dir"] =\
+ f"abfss://{resource_prefix}fs@{resource_prefix}dls.dfs.core.windows.net/{project_name}"
+
+ for k, v in DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG.items():
+ if k not in config["spark_config"]["azure_synapse"]:
+ config["spark_config"]["azure_synapse"][k] = v
def _set_databricks_config(
config: Dict,
project_name: str,
- workspace_url: str,
- cluster_name: str = None,
- use_env_vars: bool = True,
+ cluster_id: str = None,
):
- if is_databricks():
- # If this functions is being called in Databricks, we may use the context to override the provided arguments.
- ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()
- workspace_url = "https://" + ctx.tags().get("browserHostName").get()
- workspace_token = ctx.apiToken().get()
- else:
- workspace_token = os.getenv("DATABRICKS_WORKSPACE_TOKEN_VALUE", None)
-
- work_dir = f"dbfs:/{project_name}"
- databricks_config = {
- "run_name": "FEATHR_FILL_IN",
- "libraries": [{"jar": "FEATHR_FILL_IN"}],
- "spark_jar_task": {
- "main_class_name": "FEATHR_FILL_IN",
- "parameters": ["FEATHR_FILL_IN"],
- },
- }
- if cluster_name is None:
- databricks_config["new_cluster"] = DEFAULT_DATABRICKS_CLUSTER_CONFIG
- else:
- databricks_config["existing_cluster_id"] = cluster_name
- config_template = json.dumps(databricks_config)
+ if "databricks" not in config["spark_config"]:
+ config["spark_config"]["databricks"] = dict()
+
+ if "work_dir" not in config["spark_config"]["databricks"]:
+ config["spark_config"]["databricks"]["work_dir"] = f"dbfs:/{project_name}"
+
+ if "config_template" not in config["spark_config"]["databricks"]:
+ databricks_config = {
+ "run_name": "FEATHR_FILL_IN",
+ "libraries": [{"jar": "FEATHR_FILL_IN"}],
+ "spark_jar_task": {
+ "main_class_name": "FEATHR_FILL_IN",
+ "parameters": ["FEATHR_FILL_IN"],
+ },
+ }
+ if cluster_id is None:
+ databricks_config["new_cluster"] = DEFAULT_DATABRICKS_CLUSTER_CONFIG
+ else:
+ databricks_config["existing_cluster_id"] = cluster_id
- if use_env_vars:
- work_dir = os.getenv("SPARK_CONFIG__DATABRICKS__WORK_DIR", work_dir)
- workspace_url = os.getenv("SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL", workspace_url)
- workspace_token = os.getenv("DATABRICKS_WORKSPACE_TOKEN_VALUE", workspace_token)
- config_template = os.getenv("SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE", config_template)
-
- if not workspace_url:
- raise ValueError("Databricks workspace url is not provided.")
-
- if not workspace_token:
- raise ValueError("Databricks workspace token is not provided.")
-
- os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = workspace_token
- config["spark_config"]["databricks"] = {
- "work_dir": work_dir,
- "workspace_instance_url": workspace_url,
- "config_template": config_template,
- }
+ config["spark_config"]["databricks"]["config_template"] = json.dumps(databricks_config)
+
+
+def _config_kwargs_to_dict(**kwargs) -> Dict:
+ """Parse config's keyword arguments to dictionary.
+ e.g. `spark_config__spark_cluster="local"` will be parsed to `{"spark_config": {"spark_cluster": "local"}}`.
+ """
+ config = dict()
+
+ for conf_key, conf_value in kwargs.items():
+ if conf_value is None:
+ continue
+
+ conf = config
+ keys = conf_key.split("__")
+ for k in keys[:-1]:
+ if k not in conf:
+ conf[k] = dict()
+ conf = conf[k]
+ conf[keys[-1]] = conf_value
+
+ return config
+
+
+def _update_config(config: Dict, new_config: Dict):
+ """Update config dictionary with the values in `new_config`."""
+ for k, v in new_config.items():
+ if k in config and isinstance(v, collections.abc.Mapping):
+ _update_config(config[k], v)
+ else:
+ config[k] = v
+
+
+def _verify_config(config: Dict):
+ """Verify config."""
+ if config["spark_config"]["spark_cluster"] == "azure_synapse":
+ if "ADLS_KEY" not in os.environ:
+ raise ValueError("ADLS_KEY must be set in environment variables")
+
+ elif config["spark_config"]["spark_cluster"] == "databricks":
+ if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
+ raise ValueError("Databricks workspace token is not provided.")
+ elif "workspace_instance_url" not in config["spark_config"]["databricks"]:
+ raise ValueError("Databricks workspace url is not provided.")
+
+
+def _maybe_update_config_with_env_var(config: Dict, env_var_name: str):
+ """Update config dictionary with the values in environment variables.
+ e.g. `SPARK_CONFIG__SPARK_CLUSTER` will be parsed to `{"spark_config": {"spark_cluster": "local"}}`.
+ """
+ if env_var_name not in os.environ:
+ return
+
+ keys = env_var_name.lower().split("__")
+ conf = config
+ for k in keys[:-1]:
+ if k not in conf:
+ conf[k] = dict()
+ conf = conf[k]
+
+ conf[keys[-1]] = os.environ[env_var_name]
diff --git a/feathr_project/test/unit/utils/test_config.py b/feathr_project/test/unit/utils/test_config.py
index 7b3395bc9..6a5119981 100644
--- a/feathr_project/test/unit/utils/test_config.py
+++ b/feathr_project/test/unit/utils/test_config.py
@@ -18,19 +18,14 @@ def test__generate_config__output_filepath(
):
resource_prefix = "test_prefix"
project_name = "test_project"
- spark_cluster = "local"
# Use tmp_path so that the test files get cleaned up after the tests
if output_filepath:
output_filepath = str(tmp_path / output_filepath)
- if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
- os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = "test_token"
-
config_filepath = generate_config(
resource_prefix=resource_prefix,
project_name=project_name,
- spark_cluster=spark_cluster,
output_filepath=output_filepath,
use_env_vars=False,
)
@@ -45,63 +40,60 @@ def test__generate_config__output_filepath(
assert config["project_config"]["project_name"] == project_name
assert config["feature_registry"]["api_endpoint"] == f"https://{resource_prefix}webapp.azurewebsites.net/api/v1"
- assert config["spark_config"]["spark_cluster"] == spark_cluster
+ assert config["spark_config"]["spark_cluster"] == "local"
assert config["online_store"]["redis"]["host"] == f"{resource_prefix}redis.redis.cache.windows.net"
@pytest.mark.parametrize(
- "spark_cluster,cluster_name,databricks_url",
+ "spark_cluster,env_key,databricks_url",
[
("local", None, None),
- ("databricks", None, "https://test_url"),
- ("azure_synapse", "some_spark_pool", None),
+ ("databricks", "DATABRICKS_WORKSPACE_TOKEN_VALUE", "https://test_url"),
+ ("azure_synapse", "ADLS_KEY", None),
]
)
def test__generate_config__spark_cluster(
spark_cluster: str,
- cluster_name: str,
+ env_key: str,
databricks_url: str,
):
"""Test if spark cluster specific configs are generated without errors.
- TODO - For now, this test doesn't check if the config values are correct.
+ TODO - For now, this test doesn't check if the config values are correctly working with the actual Feathr client.
"""
- if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
- os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = "test_token"
+ if env_key and env_key not in os.environ:
+ os.environ[env_key] = "test_value"
generate_config(
resource_prefix="test_prefix",
project_name="test_project",
- spark_cluster=spark_cluster,
- cluster_name=cluster_name,
- databricks_url=databricks_url,
+ spark_config__spark_cluster=spark_cluster,
+ spark_config__databricks__workspace_instance_url=databricks_url,
use_env_vars=False,
)
@pytest.mark.parametrize(
- "spark_cluster,cluster_name,databricks_url",
+ "spark_cluster,env_key,databricks_url",
[
- ("databricks", "some_cluster_id", None),
- ("azure_synapse", None, "https://test_url"),
+ ("databricks", "DATABRICKS_WORKSPACE_TOKEN_VALUE", None),
]
)
def test__generate_config__exceptions(
spark_cluster: str,
- cluster_name: str,
+ env_key: str,
databricks_url: str,
):
"""Test if exceptions are raised when databricks url and token are not provided."""
- if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
- os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = "test_token"
+ if env_key and env_key not in os.environ:
+ os.environ[env_key] = "test_value"
with pytest.raises(ValueError):
generate_config(
resource_prefix="test_prefix",
project_name="test_project",
- spark_cluster=spark_cluster,
- cluster_name=cluster_name,
- databricks_url=databricks_url,
+ spark_config__spark_cluster=spark_cluster,
+ spark_config__databricks__workspace_instance_url=databricks_url,
use_env_vars=False,
)
From 8a610ac926239cb708484db999e569ebf18dcbab Mon Sep 17 00:00:00 2001
From: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
Date: Wed, 9 Nov 2022 07:01:24 +0000
Subject: [PATCH 15/18] Add more pytests
Signed-off-by: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
---
.../databricks_quickstart_nyc_taxi_demo.ipynb | 3 +-
docs/samples/nyc_taxi_demo.ipynb | 54 +++++--
feathr_project/feathr/utils/config.py | 42 ++++--
feathr_project/feathr/utils/job_utils.py | 11 +-
feathr_project/test/conftest.py | 9 +-
feathr_project/test/samples/test_notebooks.py | 6 +-
feathr_project/test/unit/utils/test_config.py | 135 ++++++++++++++----
.../test/unit/utils/test_job_utils.py | 50 +++++--
8 files changed, 233 insertions(+), 77 deletions(-)
diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb
index e562ec5db..65e305e8f 100644
--- a/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb
+++ b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb
@@ -113,8 +113,7 @@
},
"outputs": [],
"source": [
- "from datetime import datetime, timedelta\n",
- "from math import sqrt\n",
+ "from datetime import timedelta\n",
"import os\n",
"from pathlib import Path\n",
"\n",
diff --git a/docs/samples/nyc_taxi_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb
index 10e189251..4cb6b5b4f 100644
--- a/docs/samples/nyc_taxi_demo.ipynb
+++ b/docs/samples/nyc_taxi_demo.ipynb
@@ -154,13 +154,20 @@
"# Currently support: 'azure_synapse', 'databricks', and 'local' \n",
"SPARK_CLUSTER = \"local\"\n",
"\n",
- "# If \"databricks\":\n",
+ "# TODO fill values to use databricks cluster:\n",
"DATABRICKS_CLUSTER_ID = None # Set Databricks cluster id to use an existing cluster\n",
"DATABRICKS_URL = None # Set Databricks workspace url to use databricks\n",
"\n",
+ "# TODO fill values to use Azure Synapse cluster:\n",
+ "AZURE_SYNAPSE_SPARK_POOL = None # Set Azure Synapse Spark pool name\n",
+ "AZURE_SYNAPSE_URL = None # Set Azure Synapse workspace url to use Azure Synapse\n",
+ "\n",
"# Data store root path. Could be a local file system path, dbfs or Azure storage path like abfs or wasbs\n",
"DATA_STORE_PATH = TemporaryDirectory().name\n",
"\n",
+ "# Feathr config file path to use an existing file\n",
+ "FEATHR_CONFIG_PATH = None\n",
+ "\n",
"# If set True, use an interactive browser authentication to get the redis password.\n",
"USE_CLI_AUTH = False\n",
"\n",
@@ -182,7 +189,27 @@
"\n",
"`os.environ[\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"] = your-token`\n",
"\n",
- "If you are running this notebook on Databricks, the token will be automatically retrieved by using the current Databricks notebook context."
+ "If you are running this notebook on Databricks, the token will be automatically retrieved by using the current Databricks notebook context.\n",
+ "\n",
+ "On the other hand, to use Azure Synapse cluster, you have to specify the synapse workspace storage key:\n",
+ "\n",
+ "`export ADLS_KEY=your-key`\n",
+ "\n",
+ "or in the notebook cell,\n",
+ "\n",
+ "`os.environ[\"ADLS_KEY\"] = your-key`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if SPARK_CLUSTER == \"azure_synapse\" and not os.environ.get(\"ADLS_KEY\"):\n",
+ " os.environ[\"ADLS_KEY\"] = add_your_key_here\n",
+ "elif SPARK_CLUSTER == \"databricks\" and not os.environ.get(\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"):\n",
+ " os.environ[\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"] = add_your_token_here"
]
},
{
@@ -254,13 +281,18 @@
},
"outputs": [],
"source": [
- "config_path = generate_config(\n",
- " resource_prefix=RESOURCE_PREFIX,\n",
- " project_name=PROJECT_NAME,\n",
- " spark_cluster=SPARK_CLUSTER,\n",
- " databricks_cluster_id=DATABRICKS_CLUSTER_ID if SPARK_CLUSTER == \"databricks\" else None,\n",
- " spark_config__databricks__workspace_instance_url=DATABRICKS_URL,\n",
- ")\n",
+ "if FEATHR_CONFIG_PATH:\n",
+ " config_path = FEATHR_CONFIG_PATH\n",
+ "else:\n",
+ " config_path = generate_config(\n",
+ " resource_prefix=RESOURCE_PREFIX,\n",
+ " project_name=PROJECT_NAME,\n",
+ " spark_config__spark_cluster=SPARK_CLUSTER,\n",
+ " spark_config__azure_synapse__dev_url=AZURE_SYNAPSE_URL,\n",
+ " spark_config__azure_synapse__pool_name=AZURE_SYNAPSE_SPARK_POOL,\n",
+ " spark_config__databricks__workspace_instance_url=DATABRICKS_URL,\n",
+ " databricks_cluster_id=DATABRICKS_CLUSTER_ID,\n",
+ " )\n",
"\n",
"with open(config_path, 'r') as f: \n",
" print(f.read())"
@@ -334,7 +366,7 @@
" .getOrCreate()\n",
" )\n",
" \n",
- "# Else, you must already have a spark session object available in databricks or synapse."
+ "# Else, you must already have a spark session object available in databricks or synapse notebooks."
]
},
{
@@ -476,7 +508,7 @@
" # If the notebook is running on databricks, DATA_FILE_PATH should be already a dbfs path.\n",
" data_source_path = DATA_FILE_PATH\n",
"else:\n",
- " # Otherwise, upload the local file to dbfs.\n",
+ " # Otherwise, upload the local file to the cloud storage (either dbfs or adls).\n",
" data_source_path = client.feathr_spark_launcher.upload_or_get_cloud_path(DATA_FILE_PATH) "
]
},
diff --git a/feathr_project/feathr/utils/config.py b/feathr_project/feathr/utils/config.py
index 27b41e8c7..9a5f5fd89 100644
--- a/feathr_project/feathr/utils/config.py
+++ b/feathr_project/feathr/utils/config.py
@@ -48,7 +48,6 @@
DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG = {
"executor_size": "Small",
"executor_num": 2,
- "pool_name": "spark3",
}
@@ -161,20 +160,19 @@ def _set_azure_synapse_config(
resource_prefix: str,
project_name: str,
):
- """Set environment variables for Azure Synapse spark cluster."""
+ """Set configs for Azure Synapse spark cluster."""
- if "azure_synapse" not in config["spark_config"]:
- config["spark_config"]["azure_synapse"] = dict()
+ config["spark_config"]["azure_synapse"] = config["spark_config"].get("azure_synapse", {})
- if "dev_url" not in config["spark_config"]["azure_synapse"]:
+ if not config["spark_config"]["azure_synapse"].get("dev_url"):
config["spark_config"]["azure_synapse"]["dev_url"] = f"https://{resource_prefix}syws.dev.azuresynapse.net"
- if "workspace_dir" not in config["spark_config"]["azure_synapse"]:
+ if not config["spark_config"]["azure_synapse"].get("workspace_dir"):
config["spark_config"]["azure_synapse"]["workspace_dir"] =\
f"abfss://{resource_prefix}fs@{resource_prefix}dls.dfs.core.windows.net/{project_name}"
for k, v in DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG.items():
- if k not in config["spark_config"]["azure_synapse"]:
+ if not config["spark_config"]["azure_synapse"].get(k):
config["spark_config"]["azure_synapse"][k] = v
@@ -183,13 +181,14 @@ def _set_databricks_config(
project_name: str,
cluster_id: str = None,
):
- if "databricks" not in config["spark_config"]:
- config["spark_config"]["databricks"] = dict()
+ """Set configs for Databricks spark cluster."""
- if "work_dir" not in config["spark_config"]["databricks"]:
+ config["spark_config"]["databricks"] = config["spark_config"].get("databricks", {})
+
+ if not config["spark_config"]["databricks"].get("work_dir"):
config["spark_config"]["databricks"]["work_dir"] = f"dbfs:/{project_name}"
- if "config_template" not in config["spark_config"]["databricks"]:
+ if not config["spark_config"]["databricks"].get("config_template"):
databricks_config = {
"run_name": "FEATHR_FILL_IN",
"libraries": [{"jar": "FEATHR_FILL_IN"}],
@@ -239,13 +238,26 @@ def _update_config(config: Dict, new_config: Dict):
def _verify_config(config: Dict):
"""Verify config."""
if config["spark_config"]["spark_cluster"] == "azure_synapse":
- if "ADLS_KEY" not in os.environ:
+ if not os.environ.get("ADLS_KEY"):
raise ValueError("ADLS_KEY must be set in environment variables")
+ elif (
+ not os.environ.get("SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL") and
+ config["spark_config"]["azure_synapse"].get("dev_url") is None
+ ):
+ raise ValueError("Azure Synapse dev endpoint is not provided.")
+ elif (
+ not os.environ.get("SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME") and
+ config["spark_config"]["azure_synapse"].get("pool_name") is None
+ ):
+ raise ValueError("Azure Synapse pool name is not provided.")
elif config["spark_config"]["spark_cluster"] == "databricks":
- if "DATABRICKS_WORKSPACE_TOKEN_VALUE" not in os.environ:
+ if not os.environ.get("DATABRICKS_WORKSPACE_TOKEN_VALUE"):
raise ValueError("Databricks workspace token is not provided.")
- elif "workspace_instance_url" not in config["spark_config"]["databricks"]:
+ elif (
+ not os.environ.get("SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL") and
+ config["spark_config"]["databricks"].get("workspace_instance_url") is None
+ ):
raise ValueError("Databricks workspace url is not provided.")
@@ -253,7 +265,7 @@ def _maybe_update_config_with_env_var(config: Dict, env_var_name: str):
"""Update config dictionary with the values in environment variables.
e.g. `SPARK_CONFIG__SPARK_CLUSTER` will be parsed to `{"spark_config": {"spark_cluster": "local"}}`.
"""
- if env_var_name not in os.environ:
+ if not os.environ.get(env_var_name):
return
keys = env_var_name.lower().split("__")
diff --git a/feathr_project/feathr/utils/job_utils.py b/feathr_project/feathr/utils/job_utils.py
index 6f5814e43..fbc16b1ff 100644
--- a/feathr_project/feathr/utils/job_utils.py
+++ b/feathr_project/feathr/utils/job_utils.py
@@ -120,11 +120,12 @@ def get_result_df(
logger.info(f"{res_url} files will be downloaded into {local_cache_path}")
client.feathr_spark_launcher.download_result(result_path=res_url, local_folder=local_cache_path)
- # use user provided format, if there isn't one, then otherwise use the one provided by the job;
- # if none of them is available, "avro" is the default format.
- data_format: str = data_format or client.get_job_tags().get(OUTPUT_FORMAT, "")
- if data_format is None or data_format == "":
- data_format = "avro"
+ # Use the provided format or one in the job tags.
+ if data_format is None:
+ if client.get_job_tags() and client.get_job_tags().get(OUTPUT_FORMAT):
+ data_format = client.get_job_tags().get(OUTPUT_FORMAT)
+ else:
+ raise ValueError("Cannot determine the data format. Please provide the data_format argument.")
result_df = None
diff --git a/feathr_project/test/conftest.py b/feathr_project/test/conftest.py
index 52b10cf89..c2699e871 100644
--- a/feathr_project/test/conftest.py
+++ b/feathr_project/test/conftest.py
@@ -11,13 +11,16 @@ def pytest_addoption(parser):
`python -m pytest feathr_project/test/ --resource-prefix your_feathr_resource_prefix`
"""
parser.addoption(
- "--resource-prefix", action="store", default="feathrazuretest3", help="Test Azure resource prefix"
+ "--config-path",
+ action="store",
+ default=str(Path(__file__).parent.resolve().joinpath("test_user_workspace", "feathr_config.yaml")),
+ help="Test config path",
)
@pytest.fixture
-def resource_prefix(request):
- return request.config.getoption("--resource-prefix")
+def config_path(request):
+ return request.config.getoption("--config-path")
@pytest.fixture(scope="session")
diff --git a/feathr_project/test/samples/test_notebooks.py b/feathr_project/test/samples/test_notebooks.py
index f87cbff2e..c8d1cbefc 100644
--- a/feathr_project/test/samples/test_notebooks.py
+++ b/feathr_project/test/samples/test_notebooks.py
@@ -23,7 +23,7 @@
@pytest.mark.notebooks
-def test__nyc_taxi_demo(resource_prefix, tmp_path):
+def test__nyc_taxi_demo(config_path, tmp_path):
notebook_name = "nyc_taxi_demo"
output_tmpdir = TemporaryDirectory()
@@ -36,10 +36,8 @@ def test__nyc_taxi_demo(resource_prefix, tmp_path):
output_path=output_notebook_path,
# kernel_name="python3",
parameters=dict(
- RESOURCE_PREFIX=resource_prefix,
- PROJECT_NAME=notebook_name,
+ FEATHR_CONFIG_PATH=config_path,
DATA_STORE_PATH=output_tmpdir.name,
- SPARK_CLUSTER="local",
USE_CLI_AUTH=False,
REGISTER_FEATURES=False,
SCRAP_RESULTS=True,
diff --git a/feathr_project/test/unit/utils/test_config.py b/feathr_project/test/unit/utils/test_config.py
index 6a5119981..770980e12 100644
--- a/feathr_project/test/unit/utils/test_config.py
+++ b/feathr_project/test/unit/utils/test_config.py
@@ -1,11 +1,13 @@
from copy import deepcopy
import os
from pathlib import Path
+from unittest.mock import MagicMock
import yaml
import pytest
+from pytest_mock import MockerFixture
-from feathr import FeathrClient
+import feathr.utils.config
from feathr.utils.config import generate_config
@@ -45,55 +47,134 @@ def test__generate_config__output_filepath(
@pytest.mark.parametrize(
- "spark_cluster,env_key,databricks_url",
+ "spark_cluster,env_key,kwargs",
[
- ("local", None, None),
- ("databricks", "DATABRICKS_WORKSPACE_TOKEN_VALUE", "https://test_url"),
- ("azure_synapse", "ADLS_KEY", None),
+ ("local", None, dict()),
+ (
+ "databricks",
+ "DATABRICKS_WORKSPACE_TOKEN_VALUE",
+ dict(spark_config__databricks__workspace_instance_url="databricks_url"),
+ ),
+ (
+ "azure_synapse",
+ "ADLS_KEY",
+ dict(
+ spark_config__azure_synapse__dev_url="synapse_url",
+ spark_config__azure_synapse__pool_name="pool_name",
+ ),
+ ),
]
)
def test__generate_config__spark_cluster(
+ mocker: MockerFixture,
spark_cluster: str,
env_key: str,
- databricks_url: str,
+ kwargs: str,
):
"""Test if spark cluster specific configs are generated without errors.
TODO - For now, this test doesn't check if the config values are correctly working with the actual Feathr client.
"""
-
- if env_key and env_key not in os.environ:
- os.environ[env_key] = "test_value"
+ # Mock the os.environ to return the specified env vars
+ mocker.patch.object(feathr.utils.config.os, "environ", {env_key: "some_value"})
generate_config(
resource_prefix="test_prefix",
project_name="test_project",
spark_config__spark_cluster=spark_cluster,
- spark_config__databricks__workspace_instance_url=databricks_url,
use_env_vars=False,
+ **kwargs,
)
@pytest.mark.parametrize(
- "spark_cluster,env_key,databricks_url",
+ "adls_key,pool_name,expected_error",
[
- ("databricks", "DATABRICKS_WORKSPACE_TOKEN_VALUE", None),
+ ("some_key", "some_name", None),
+ (None, "some_name", ValueError),
+ ("some_key", None, ValueError),
]
)
-def test__generate_config__exceptions(
- spark_cluster: str,
- env_key: str,
- databricks_url: str,
+def test__generate_config__azure_synapse_exceptions(
+ mocker: MockerFixture,
+ adls_key: str,
+ pool_name: str,
+ expected_error: Exception,
+):
+ """Test if exceptions are raised when databricks url and token are not provided."""
+
+ # Either env vars or argument should yield the same result
+ for environ in [{"ADLS_KEY": adls_key}, {
+ "ADLS_KEY": adls_key,
+ "SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME": pool_name,
+ }]:
+ # Mock the os.environ to return the specified env vars
+ mocker.patch.object(feathr.utils.config.os, "environ", environ)
+
+ # Test either using env vars or arguments
+ if "SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME" in environ:
+ kwargs = dict()
+ else:
+ kwargs = dict(spark_config__azure_synapse__pool_name=pool_name)
+
+ if expected_error is None:
+ generate_config(
+ resource_prefix="test_prefix",
+ project_name="test_project",
+ spark_config__spark_cluster="azure_synapse",
+ **kwargs,
+ )
+ else:
+ with pytest.raises(ValueError):
+ generate_config(
+ resource_prefix="test_prefix",
+ project_name="test_project",
+ spark_config__spark_cluster="azure_synapse",
+ **kwargs,
+ )
+
+
+@pytest.mark.parametrize(
+ "databricks_token,workspace_url,expected_error",
+ [
+ ("some_token", "some_url", None),
+ (None, "some_url", ValueError),
+ ("some_token", None, ValueError),
+ ]
+)
+def test__generate_config__databricks_exceptions(
+ mocker: MockerFixture,
+ databricks_token: str,
+ workspace_url: str,
+ expected_error: Exception,
):
"""Test if exceptions are raised when databricks url and token are not provided."""
- if env_key and env_key not in os.environ:
- os.environ[env_key] = "test_value"
-
- with pytest.raises(ValueError):
- generate_config(
- resource_prefix="test_prefix",
- project_name="test_project",
- spark_config__spark_cluster=spark_cluster,
- spark_config__databricks__workspace_instance_url=databricks_url,
- use_env_vars=False,
- )
+ # Either env vars or argument should yield the same result
+ for environ in [{"DATABRICKS_WORKSPACE_TOKEN_VALUE": databricks_token}, {
+ "DATABRICKS_WORKSPACE_TOKEN_VALUE": databricks_token,
+ "SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL": workspace_url,
+ }]:
+ # Mock the os.environ to return the specified env vars
+ mocker.patch.object(feathr.utils.config.os, "environ", environ)
+
+ # Test either using env vars or arguments
+ if "SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL" in environ:
+ kwargs = dict()
+ else:
+ kwargs = dict(spark_config__databricks__workspace_instance_url=workspace_url)
+
+ if expected_error is None:
+ generate_config(
+ resource_prefix="test_prefix",
+ project_name="test_project",
+ spark_config__spark_cluster="databricks",
+ **kwargs,
+ )
+ else:
+ with pytest.raises(ValueError):
+ generate_config(
+ resource_prefix="test_prefix",
+ project_name="test_project",
+ spark_config__spark_cluster="databricks",
+ **kwargs,
+ )
diff --git a/feathr_project/test/unit/utils/test_job_utils.py b/feathr_project/test/unit/utils/test_job_utils.py
index 136bc3545..9f82be66e 100644
--- a/feathr_project/test/unit/utils/test_job_utils.py
+++ b/feathr_project/test/unit/utils/test_job_utils.py
@@ -10,6 +10,7 @@
from pyspark.sql import DataFrame, SparkSession
from feathr import FeathrClient
+from feathr.constants import OUTPUT_FORMAT, OUTPUT_PATH_TAG
from feathr.utils.job_utils import (
get_result_df,
get_result_pandas_df,
@@ -86,11 +87,24 @@ def test__get_result_df__with_local_cache_path(
@pytest.mark.parametrize(
- "is_databricks,spark_runtime,res_url,expected_error", [
- (True, "local", None, RuntimeError), # Test RuntimeError when the function is running at Databricks but client.spark_runtime is not databricks
+ "is_databricks,spark_runtime,res_url,data_format,expected_error", [
+ # Test RuntimeError when the function is running at Databricks but client.spark_runtime is not databricks
+ (True, "local", "some_url", "some_format", RuntimeError),
+ (True, "azure_synapse", "some_url", "some_format", RuntimeError),
+ (True, "databricks", "some_url", "some_format", None),
+ (False, "local", "some_url", "some_format", None),
+ (False, "azure_synapse", "some_url", "some_format", None),
+ (False, "databricks", "some_url", "some_format", None),
# Test ValueError when res_url is None
- (False, "local", None, ValueError),
- (True, "databricks", None, ValueError),
+ (True, "databricks", None, "some_format", ValueError),
+ (False, "local", None, "some_format", ValueError),
+ (False, "azure_synapse", None, "some_format", ValueError),
+ (False, "databricks", None, "some_format", ValueError),
+ # Test ValueError when data_format is None
+ (True, "databricks", "some_url", None, ValueError),
+ (False, "local", "some_url", None, ValueError),
+ (False, "azure_synapse", "some_url", None, ValueError),
+ (False, "databricks", "some_url", None, ValueError),
]
)
def test__get_result_df__exceptions(
@@ -98,13 +112,10 @@ def test__get_result_df__exceptions(
is_databricks: bool,
spark_runtime: str,
res_url: str,
+ data_format: str,
expected_error: Type[Exception],
):
"""Test exceptions"""
- # Mock client
- client = MagicMock()
- client.get_job_result_uri = MagicMock(return_value=res_url)
- client.spark_runtime = spark_runtime
# Mock is_data_bricks
mocker.patch("feathr.utils.job_utils.is_databricks", return_value=is_databricks)
@@ -112,8 +123,27 @@ def test__get_result_df__exceptions(
# Mock _load_files_to_pandas_df
mocker.patch("feathr.utils.job_utils._load_files_to_pandas_df")
- with pytest.raises(expected_error):
- get_result_df(client)
+ # Either job tags or argument should yield the same result
+ for job_tag in [None, {OUTPUT_FORMAT: data_format, OUTPUT_PATH_TAG: res_url}]:
+ # Mock client
+ client = MagicMock()
+ client.get_job_result_uri = MagicMock(return_value=res_url)
+ client.get_job_tags = MagicMock(return_value=job_tag)
+ client.spark_runtime = spark_runtime
+
+ if expected_error is None:
+ get_result_df(
+ client=client,
+ res_url=None if job_tag else res_url,
+ data_format=None if job_tag else data_format,
+ )
+ else:
+ with pytest.raises(expected_error):
+ get_result_df(
+ client=client,
+ res_url=None if job_tag else res_url,
+ data_format=None if job_tag else data_format,
+ )
@pytest.mark.parametrize(
From 4c50485a725685534e7caa7bf646cb64c4800123 Mon Sep 17 00:00:00 2001
From: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
Date: Wed, 9 Nov 2022 20:50:49 +0000
Subject: [PATCH 16/18] Use None as default dataformat in the job_utils.
Instead, set 'avro' as a default output format to the job tags from the
client
Signed-off-by: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
---
docs/samples/nyc_taxi_demo.ipynb | 6 ++-
feathr_project/feathr/client.py | 2 +
feathr_project/feathr/utils/job_utils.py | 42 +++++++++----------
.../test/test_input_output_sources.py | 19 ++++-----
4 files changed, 33 insertions(+), 36 deletions(-)
diff --git a/docs/samples/nyc_taxi_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb
index 4cb6b5b4f..b80bac374 100644
--- a/docs/samples/nyc_taxi_demo.ipynb
+++ b/docs/samples/nyc_taxi_demo.ipynb
@@ -361,11 +361,13 @@
" SparkSession\n",
" .builder\n",
" .appName(\"feathr\")\n",
- " .config(\"spark.jars.packages\", \"org.apache.spark:spark-avro_2.12:3.3.0\")\n",
+ " .config(\"spark.jars.packages\", \"org.apache.spark:spark-avro_2.12:3.3.0,io.delta:delta-core_2.12:2.1.1\")\n",
+ " .config(\"spark.sql.extensions\", \"io.delta.sql.DeltaSparkSessionExtension\")\n",
+ " .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.spark.sql.delta.catalog.DeltaCatalog\")\n",
" .config(\"spark.ui.port\", \"8080\") # Set ui port other than the default one (4040) so that feathr spark job doesn't fail. \n",
" .getOrCreate()\n",
" )\n",
- " \n",
+ "\n",
"# Else, you must already have a spark session object available in databricks or synapse notebooks."
]
},
diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py
index cd080f871..741317428 100644
--- a/feathr_project/feathr/client.py
+++ b/feathr_project/feathr/client.py
@@ -487,6 +487,8 @@ def _get_offline_features_with_config(self,
# set output format in job tags if it's set by user, so that it can be used to parse the job result in the helper function
if execution_configurations is not None and OUTPUT_FORMAT in execution_configurations:
job_tags[OUTPUT_FORMAT] = execution_configurations[OUTPUT_FORMAT]
+ else:
+ job_tags[OUTPUT_FORMAT] = "avro"
'''
- Job tags are for job metadata and it's not passed to the actual spark job (i.e. not visible to spark job), more like a platform related thing that Feathr want to add (currently job tags only have job output URL and job output format, ). They are carried over with the job and is visible to every Feathr client. Think this more like some customized metadata for the job which would be weird to be put in the spark job itself.
- Job arguments (or sometimes called job parameters)are the arguments which are command line arguments passed into the actual spark job. This is usually highly related with the spark job. In Feathr it's like the input to the scala spark CLI. They are usually not spark specific (for example if we want to specify the location of the feature files, or want to
diff --git a/feathr_project/feathr/utils/job_utils.py b/feathr_project/feathr/utils/job_utils.py
index fbc16b1ff..d9c73c355 100644
--- a/feathr_project/feathr/utils/job_utils.py
+++ b/feathr_project/feathr/utils/job_utils.py
@@ -22,9 +22,9 @@ def get_result_pandas_df(
Args:
client: Feathr client
data_format: Format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`.
- Default to `avro` if not specified.
+ Default to use client's job tags if exists.
res_url: Result URL to download files from. Note that this will not block the job so you need to make sure
- the job is finished and the result URL contains actual data.
+ the job is finished and the result URL contains actual data. Default to use client's job tags if exists.
local_cache_path (optional): Specify the absolute download path. if the user does not provide this,
the function will create a temporary directory.
@@ -47,9 +47,9 @@ def get_result_spark_df(
spark: Spark session
client: Feathr client
data_format: Format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`.
- Default to `avro` if not specified.
+ Default to use client's job tags if exists.
res_url: Result URL to download files from. Note that this will not block the job so you need to make sure
- the job is finished and the result URL contains actual data.
+ the job is finished and the result URL contains actual data. Default to use client's job tags if exists.
local_cache_path (optional): Specify the absolute download path. if the user does not provide this,
the function will create a temporary directory.
@@ -71,9 +71,9 @@ def get_result_df(
Args:
client: Feathr client
data_format: Format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`.
- Default to `avro` if not specified.
+ Default to use client's job tags if exists.
res_url: Result URL to download files from. Note that this will not block the job so you need to make sure
- the job is finished and the result URL contains actual data.
+ the job is finished and the result URL contains actual data. Default to use client's job tags if exists.
local_cache_path (optional): Specify the absolute download directory. if the user does not provide this,
the function will create a temporary directory.
spark (optional): Spark session. If provided, the function returns spark Dataframe.
@@ -82,9 +82,22 @@ def get_result_df(
Returns:
Either Spark or pandas DataFrame.
"""
+ if data_format is None:
+ # May use data format from the job tags
+ if client.get_job_tags() and client.get_job_tags().get(OUTPUT_FORMAT):
+ data_format = client.get_job_tags().get(OUTPUT_FORMAT)
+ else:
+ raise ValueError("Cannot determine the data format. Please provide the data_format argument.")
+
+ data_format = data_format.lower()
+
if is_databricks() and client.spark_runtime != "databricks":
raise RuntimeError(f"The function is called from Databricks but the client.spark_runtime is {client.spark_runtime}.")
+ # TODO Loading Synapse Delta table result into pandas has a bug: https://github.com/delta-io/delta-rs/issues/582
+ if not spark and client.spark_runtime == "azure_synapse" and data_format == "delta":
+ raise RuntimeError(f"Loading Delta table result from Azure Synapse into pandas DataFrame is not supported. You maybe able to use spark DataFrame to load the result instead.")
+
# use a result url if it's provided by the user, otherwise use the one provided by the job
res_url: str = res_url or client.get_job_result_uri(block=True, timeout_sec=1200)
if res_url is None:
@@ -120,15 +133,7 @@ def get_result_df(
logger.info(f"{res_url} files will be downloaded into {local_cache_path}")
client.feathr_spark_launcher.download_result(result_path=res_url, local_folder=local_cache_path)
- # Use the provided format or one in the job tags.
- if data_format is None:
- if client.get_job_tags() and client.get_job_tags().get(OUTPUT_FORMAT):
- data_format = client.get_job_tags().get(OUTPUT_FORMAT)
- else:
- raise ValueError("Cannot determine the data format. Please provide the data_format argument.")
-
result_df = None
-
try:
if spark is not None:
if data_format == "csv":
@@ -154,17 +159,8 @@ def _load_files_to_pandas_df(dir_path: str, data_format: str = "avro") -> pd.Dat
elif data_format == "delta":
from deltalake import DeltaTable
-
delta = DeltaTable(dir_path)
- # if client.spark_runtime != "azure_synapse":
- # don't detect for synapse result with Delta as there's a problem with underlying system
- # Issues are tracked here: https://github.com/delta-io/delta-rs/issues/582
return delta.to_pyarrow_table().to_pandas()
- # else:
- # TODO -- Proper warning messages. Is this applied to all the other formats?
- # raise RuntimeError(
- # "Please use Azure Synapse to read the result in the Azure Synapse cluster. Reading local results is not supported for Azure Synapse."
- # )
elif data_format == "avro":
import pandavro as pdx
diff --git a/feathr_project/test/test_input_output_sources.py b/feathr_project/test/test_input_output_sources.py
index f4af85678..ba4b3921a 100644
--- a/feathr_project/test/test_input_output_sources.py
+++ b/feathr_project/test/test_input_output_sources.py
@@ -10,6 +10,7 @@
from test_fixture import basic_test_setup
from test_utils.constants import Constants
+
# test parquet file read/write without an extension name
def test_feathr_get_offline_features_with_parquet():
"""
@@ -38,7 +39,7 @@ def test_feathr_get_offline_features_with_parquet():
else:
output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), ".parquet"])
-
+
client.get_offline_features(observation_settings=settings,
feature_query=feature_query,
output_path=output_path,
@@ -47,14 +48,12 @@ def test_feathr_get_offline_features_with_parquet():
# assuming the job can successfully run; otherwise it will throw exception
client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS)
-
+
# download result and just assert the returned result is not empty
res_df = get_result_df(client)
assert res_df.shape[0] > 0
-
-
# test delta lake read/write without an extension name
def test_feathr_get_offline_features_with_delta_lake():
"""
@@ -83,7 +82,7 @@ def test_feathr_get_offline_features_with_delta_lake():
else:
output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), "_deltalake"])
-
+
client.get_offline_features(observation_settings=settings,
feature_query=feature_query,
output_path=output_path,
@@ -92,15 +91,13 @@ def test_feathr_get_offline_features_with_delta_lake():
# assuming the job can successfully run; otherwise it will throw exception
client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS)
-
+
# wait for a few secs for the resource to come up in the databricks API
time.sleep(5)
- # download result and just assert the returned result is not empty
- res_df = get_result_df(client)
-
+ # download result and just assert the returned result is not empty
+ # if users are using delta format in synapse, skip this check, due to issue https://github.com/delta-io/delta-rs/issues/582
result_format: str = client.get_job_tags().get(OUTPUT_FORMAT, "")
if not (client.spark_runtime == 'azure_synapse' and result_format == 'delta'):
- # if users are using delta format in synapse, skip this check, due to issue https://github.com/delta-io/delta-rs/issues/582
+ res_df = get_result_df(client)
assert res_df.shape[0] > 0
-
From c049958b18910f9fb6e32d58087b8b1bb0704893 Mon Sep 17 00:00:00 2001
From: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
Date: Thu, 10 Nov 2022 00:00:43 +0000
Subject: [PATCH 17/18] Change feathr client to mocked object
Signed-off-by: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
---
.../test/unit/utils/test_job_utils.py | 74 ++++++++++---------
1 file changed, 40 insertions(+), 34 deletions(-)
diff --git a/feathr_project/test/unit/utils/test_job_utils.py b/feathr_project/test/unit/utils/test_job_utils.py
index 9f82be66e..0909fb56e 100644
--- a/feathr_project/test/unit/utils/test_job_utils.py
+++ b/feathr_project/test/unit/utils/test_job_utils.py
@@ -157,30 +157,35 @@ def test__get_result_df__exceptions(
)
def test__get_result_df(
workspace_dir: str,
- feathr_client: FeathrClient,
data_format: str,
output_filename: str,
expected_count: int,
):
"""Test get_result_df returns pandas DataFrame"""
- # Note: make sure the output file exists in the test_user_workspace
- res_url = str(Path(workspace_dir, "mock_results", output_filename))
- local_cache_path = res_url
+ for spark_runtime in ["local", "databricks", "azure_synapse"]:
+ # Note: make sure the output file exists in the test_user_workspace
+ res_url = str(Path(workspace_dir, "mock_results", output_filename))
+ local_cache_path = res_url
- # Mock feathr_spark_launcher.download_result
- feathr_client.feathr_spark_launcher.download_result = MagicMock()
+ # Mock client
+ client = MagicMock()
+ client.spark_runtime = spark_runtime
- if feathr_client.spark_runtime == "databricks":
- res_url = f"dbfs:/{res_url}"
+ # Mock feathr_spark_launcher.download_result
+ if client.spark_runtime == "databricks":
+ res_url = f"dbfs:/{res_url}"
+ if client.spark_runtime == "azure_synapse" and data_format == "delta":
+ # TODO currently pass the delta table test on Synapse result due to the delta table package bug.
+ continue
- df = get_result_df(
- client=feathr_client,
- data_format=data_format,
- res_url=res_url,
- local_cache_path=local_cache_path,
- )
- assert isinstance(df, pd.DataFrame)
- assert len(df) == expected_count
+ df = get_result_df(
+ client=client,
+ data_format=data_format,
+ res_url=res_url,
+ local_cache_path=local_cache_path,
+ )
+ assert isinstance(df, pd.DataFrame)
+ assert len(df) == expected_count
@pytest.mark.parametrize(
@@ -194,29 +199,30 @@ def test__get_result_df(
)
def test__get_result_df__with_spark_session(
workspace_dir: str,
- feathr_client: FeathrClient,
spark: SparkSession,
data_format: str,
output_filename: str,
expected_count: int,
):
"""Test get_result_df returns spark DataFrame"""
- # Note: make sure the output file exists in the test_user_workspace
- res_url = str(Path(workspace_dir, "mock_results", output_filename))
- local_cache_path = res_url
-
- # Mock feathr_spark_launcher.download_result
- feathr_client.feathr_spark_launcher.download_result = MagicMock()
+ for spark_runtime in ["local", "databricks", "azure_synapse"]:
+ # Note: make sure the output file exists in the test_user_workspace
+ res_url = str(Path(workspace_dir, "mock_results", output_filename))
+ local_cache_path = res_url
- if feathr_client.spark_runtime == "databricks":
- res_url = f"dbfs:/{res_url}"
+ # Mock client
+ client = MagicMock()
+ client.spark_runtime = spark_runtime
- df = get_result_df(
- client=feathr_client,
- data_format=data_format,
- res_url=res_url,
- spark=spark,
- local_cache_path=local_cache_path,
- )
- assert isinstance(df, DataFrame)
- assert df.count() == expected_count
+ if client.spark_runtime == "databricks":
+ res_url = f"dbfs:/{res_url}"
+
+ df = get_result_df(
+ client=client,
+ data_format=data_format,
+ res_url=res_url,
+ spark=spark,
+ local_cache_path=local_cache_path,
+ )
+ assert isinstance(df, DataFrame)
+ assert df.count() == expected_count
From 190377c1b52e98d3ef1b5a0d5516c52af94f22ab Mon Sep 17 00:00:00 2001
From: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
Date: Tue, 22 Nov 2022 20:04:10 +0000
Subject: [PATCH 18/18] Change timeout to 1000s in the notebook
Signed-off-by: Jun Ki Min <42475935+loomlike@users.noreply.github.com>
---
docs/samples/nyc_taxi_demo.ipynb | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/samples/nyc_taxi_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb
index 81a11a460..31754950e 100644
--- a/docs/samples/nyc_taxi_demo.ipynb
+++ b/docs/samples/nyc_taxi_demo.ipynb
@@ -751,7 +751,7 @@
" output_path=offline_features_path,\n",
")\n",
"\n",
- "client.wait_job_to_finish(timeout_sec=500)"
+ "client.wait_job_to_finish(timeout_sec=1000)"
]
},
{
@@ -1020,7 +1020,7 @@
" execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n",
")\n",
"\n",
- "client.wait_job_to_finish(timeout_sec=500)"
+ "client.wait_job_to_finish(timeout_sec=1000)"
]
},
{