diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21ba0cd936310..f7e67285326ae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -477,13 +477,6 @@ If contributing to MLflow's R APIs, install [R](https://cloud.r-project.org/) and make sure that you have satisfied all the [Environment Setup and Python configuration](#environment-setup-and-python-configuration). -For changes to R documentation, also install -[pandoc](https://pandoc.org/installing.html) 2.2.1 or above, verifying -the version of your installation via `pandoc --version`. If using Mac -OSX, note that the homebrew installation of pandoc may be out of date - -you can find newer pandoc versions at -. - The `mlflow/R/mlflow` directory contains R wrappers for the Projects, Tracking and Models components. These wrappers depend on the Python package, so first install the Python package in a conda environment: @@ -836,6 +829,16 @@ python setup.py bdist_wheel First, install dependencies for building docs as described in [Environment Setup and Python configuration](#environment-setup-and-python-configuration). +Building documentation requires [Pandoc](https://pandoc.org/index.html). It should have already been +installed if you used the automated env setup script +([dev-env-setup.sh](https://github.com/mlflow/mlflow/blob/master/dev/dev-env-setup.sh)), +but if you are manually installing dependencies, please follow [the official instruction](https://pandoc.org/installing.html). + +Also, check the version of your installation via `pandoc --version` and ensure it is 2.2.1 or above. +If you are using Mac OSX, be aware that the Homebrew installation of Pandoc may be outdated. If you are using Linux, +you should use a deb installer or install from the source, instead of running `apt` / `apt-get` commands. Pandoc package available on official +repositories is an older version and contains several bugs. You can find newer versions at . + To generate a live preview of Python & other rst documentation, run the following snippet. Note that R & Java API docs must be regenerated separately after each change and are not live-updated; see subsequent diff --git a/dev/dev-env-setup.sh b/dev/dev-env-setup.sh index fbb2616934faf..e04aa2f868c54 100755 --- a/dev/dev-env-setup.sh +++ b/dev/dev-env-setup.sh @@ -105,6 +105,45 @@ minor_to_micro() { esac } +# Check if brew is installed and install it if it isn't present +# Note: if xcode isn't installed, this will fail. +# $1: name of package that requires brew +check_and_install_brew() { + if [ -z "$(command -v brew)" ]; then + echo "Homebrew is required to install $1 on MacOS. Installing in your home directory." + bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + fi + echo "Updating brew..." + brew update +} + +# Compare two version numbers +# Usage: version_gt version1 version2 +# Returns 0 (true) if version1 > version2, 1 (false) otherwise +version_gt() { + IFS='.' read -ra VER1 <<< "$1" + IFS='.' read -ra VER2 <<< "$2" + + # Compare each segment of the version numbers + for (( i=0; i<"${#VER1[@]}"; i++ )); do + # If VER2 is shorter and we haven't found a difference yet, VER1 is greater + if [[ -z ${VER2[i]} ]]; then + return 0 + fi + + # If some segments are not equal, return their comparison result + if (( ${VER1[i]} > ${VER2[i]} )); then + return 0 + elif (( ${VER1[i]} < ${VER2[i]} )); then + return 1 + fi + done + + # If all common length segments are same, the one with more segments is greater + return $(( ${#VER1[@]} <= ${#VER2[@]} )) +} + + # Check if pyenv is installed and offer to install it if not present pyenv_exist=$(command -v pyenv) @@ -115,15 +154,9 @@ if [ -z "$pyenv_exist" ]; then fi if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then if [[ "$machine" == mac ]]; then - # Check if brew is installed and install it if it isn't present - # Note: if xcode isn't installed, this will fail. - if [ -z "$(command -v brew)" ]; then - echo "Homebrew is required to install pyenv on MacOS. Installing in your home directory." - bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - fi - echo "Updating brew and installing pyenv..." + check_and_install_brew "pyenv" + echo "Installing pyenv..." echo "Note: this will probably take a considerable amount of time." - brew update brew install pyenv brew install openssl readline sqlite3 xz zlib elif [[ "$machine" == linux ]]; then @@ -242,6 +275,34 @@ echo "$(pip freeze)$(tput sgr0)" command -v docker >/dev/null 2>&1 || echo "$(tput bold; tput setaf 1)A docker installation cannot be found. Please ensure that docker is installed to run all tests locally.$(tput sgr0)" + +# check if pandoc with required version is installed and offer to install it if not present +pandoc_version=$(pandoc --version | grep "pandoc" | awk '{print $2}') +if [[ -z "$pandoc_version" ]] || ! version_gt "$pandoc_version" "2.2.1"; then + if [ -z "$GITHUB_ACTIONS" ]; then + read -p "Pandoc version 2.2.1 or above is required to generate documentation. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r + echo + fi + + if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then + echo "Installing Pandoc..." + if [[ "$machine" == mac ]]; then + check_and_install_brew "pandoc" + brew install pandoc + elif [[ "$machine" == linux ]]; then + # install pandoc via deb package as `apt-get` gives too old version + TEMP_DEB=$(mktemp) && \ + wget --directory-prefix $TEMP_DEB https://github.com/jgm/pandoc/releases/download/2.16.2/pandoc-2.16.2-1-amd64.deb && \ + sudo dpkg --install $(find $TEMP_DEB -name '*.deb') && \ + rm -rf $TEMP_DEB + else + echo "Unknown operating system environment: $machine exiting." + exit 1 + fi + fi +fi + + # Setup git environment configuration for proper signing of commits git_user=$(git config user.name) git_email=$(git config user.email) @@ -265,4 +326,4 @@ fi # setup pre-commit hooks pre-commit install -t pre-commit -t prepare-commit-msg -echo "$(tput setaf 2)Your MLflow development environment can be activated by running: $(tput bold)source $VENV_DIR$(tput sgr0)" +echo "$(tput setaf 2)Your MLflow development environment can be activated by running: $(tput bold)source $VENV_DIR$(tput sgr0)" \ No newline at end of file diff --git a/docs/source/deep-learning/index.rst b/docs/source/deep-learning/index.rst index cec4e6c2f93ce..3ea423d6c54e4 100644 --- a/docs/source/deep-learning/index.rst +++ b/docs/source/deep-learning/index.rst @@ -47,27 +47,27 @@ The officially supported integrations for deep learning libraries in MLflow enco diff --git a/docs/source/deployment/deploy-model-to-kubernetes/index.rst b/docs/source/deployment/deploy-model-to-kubernetes/index.rst index c03dc5a5efcb3..80d961497f90e 100644 --- a/docs/source/deployment/deploy-model-to-kubernetes/index.rst +++ b/docs/source/deployment/deploy-model-to-kubernetes/index.rst @@ -126,6 +126,7 @@ Next, use the MLflow UI to compare the models that you have produced. In the sam as the one that contains the ``mlruns`` run: .. code-section:: + .. code-block:: shell mlflow ui diff --git a/docs/source/getting-started/index.rst b/docs/source/getting-started/index.rst index d99945f0d550c..03e741e09fceb 100644 --- a/docs/source/getting-started/index.rst +++ b/docs/source/getting-started/index.rst @@ -30,7 +30,7 @@ If you would like to get started immediately by interactively running the notebo .. raw:: html - Download the Notebook
+ Download the Notebook
Quickstart elements ^^^^^^^^^^^^^^^^^^^ @@ -76,7 +76,7 @@ If you would like to get started immediately by interactively running the notebo .. raw:: html - Download the Notebook
+ Download the Notebook
Guide sections ^^^^^^^^^^^^^^ diff --git a/docs/source/getting-started/intro-quickstart/index.rst b/docs/source/getting-started/intro-quickstart/index.rst index a8302ddb82379..035c4bf7ca0d5 100644 --- a/docs/source/getting-started/intro-quickstart/index.rst +++ b/docs/source/getting-started/intro-quickstart/index.rst @@ -41,6 +41,7 @@ Step 1 - Get MLflow MLflow is available on PyPI. If you don't already have it installed on your system, you can install it with: .. code-section:: + .. code-block:: bash :name: download-mlflow @@ -53,6 +54,7 @@ We're going to start a local MLflow Tracking Server, which we will connect to fo From a terminal, run: .. code-section:: + .. code-block:: bash :name: tracking-server-start @@ -72,6 +74,7 @@ In this section, we're going to log a model with MLflow. A quick overview of the .. code-section:: + .. code-block:: python :name: train-model @@ -132,6 +135,7 @@ The steps that we will take are: to ensure that the loggable content (parameters, metrics, artifacts, and the model) are fully materialized prior to logging. .. code-section:: + .. code-block:: python :name: log-model @@ -177,6 +181,7 @@ After logging the model, we can perform inference by: below. .. code-section:: + .. code-block:: python :name: load-model diff --git a/docs/source/getting-started/logging-first-model/step1-tracking-server.rst b/docs/source/getting-started/logging-first-model/step1-tracking-server.rst index 0b102cd6d8a65..52e2713bbec6f 100644 --- a/docs/source/getting-started/logging-first-model/step1-tracking-server.rst +++ b/docs/source/getting-started/logging-first-model/step1-tracking-server.rst @@ -15,6 +15,7 @@ Step 1: Install MLflow from PyPI MLflow is conveniently available on PyPI. Installing it is as simple as running a pip command. .. code-section:: + .. code-block:: bash :name: download-mlflow @@ -27,6 +28,7 @@ To begin, you'll need to initiate the MLflow Tracking Server. Remember to keep t running during the tutorial, as closing it will shut down the server. .. code-section:: + .. code-block:: bash :name: tracking-server-start diff --git a/docs/source/getting-started/logging-first-model/step2-mlflow-client.rst b/docs/source/getting-started/logging-first-model/step2-mlflow-client.rst index e57d35cee4fd1..0f70e76bdd4cd 100644 --- a/docs/source/getting-started/logging-first-model/step2-mlflow-client.rst +++ b/docs/source/getting-started/logging-first-model/step2-mlflow-client.rst @@ -18,6 +18,7 @@ Importing Dependencies In order to use the MLflowClient API, the initial step involves importing the necessary modules. .. code-section:: + .. code-block:: python :name: imports :emphasize-lines: 1 @@ -43,6 +44,7 @@ assigned the server when we started it. The two components that we submitted as ``host`` and the ``port``. Combined, these form the ``tracking_uri`` argument that we will specify to start an instance of the client. .. code-section:: + .. code-block:: python :name: client @@ -70,6 +72,7 @@ The first thing that we're going to do is to view the metadata associated with t use of the :py:func:`mlflow.client.MlflowClient.search_experiments` API. Let's issue a search query to see what the results are. .. code-section:: + .. code-block:: python all_experiments = client.search_experiments() @@ -91,6 +94,7 @@ To get familiar with accessing elements from returned collections from MLflow AP query and extract these attributes into a dict. .. code-section:: + .. code-block:: python default_experiment = [ diff --git a/docs/source/getting-started/logging-first-model/step3-create-experiment.rst b/docs/source/getting-started/logging-first-model/step3-create-experiment.rst index 131e5991478e7..c73cc7da240f6 100644 --- a/docs/source/getting-started/logging-first-model/step3-create-experiment.rst +++ b/docs/source/getting-started/logging-first-model/step3-create-experiment.rst @@ -105,6 +105,7 @@ Creating the Apples Experiment with Meaningful tags --------------------------------------------------- .. code-section:: + .. code-block:: python # Provide an Experiment description that will appear in the UI diff --git a/docs/source/getting-started/logging-first-model/step4-experiment-search.rst b/docs/source/getting-started/logging-first-model/step4-experiment-search.rst index 3567a9b6a3e8e..d73cd8ea1b6c0 100644 --- a/docs/source/getting-started/logging-first-model/step4-experiment-search.rst +++ b/docs/source/getting-started/logging-first-model/step4-experiment-search.rst @@ -51,6 +51,7 @@ tags, note the particular syntax used. The custom tag names are wrapped with bac condition is wrapped in single quotes. .. code-section:: + .. code-block:: python # Use search_experiments() to search on the project_name tag key diff --git a/docs/source/getting-started/logging-first-model/step5-synthetic-data.rst b/docs/source/getting-started/logging-first-model/step5-synthetic-data.rst index 8c420512df629..8d3e90d075bc7 100644 --- a/docs/source/getting-started/logging-first-model/step5-synthetic-data.rst +++ b/docs/source/getting-started/logging-first-model/step5-synthetic-data.rst @@ -21,6 +21,7 @@ We can introduce this correlation by crafting a relationship between our feature The random elements of some of the factors will handle the unexplained variance portion. .. code-section:: + .. code-block:: python import pandas as pd diff --git a/docs/source/getting-started/logging-first-model/step6-logging-a-run.rst b/docs/source/getting-started/logging-first-model/step6-logging-a-run.rst index 81b51d83897bd..55b87b4595257 100644 --- a/docs/source/getting-started/logging-first-model/step6-logging-a-run.rst +++ b/docs/source/getting-started/logging-first-model/step6-logging-a-run.rst @@ -76,6 +76,7 @@ using MLflow to tracking a training iteration. To start with, we will need to import our required modules. .. code-section:: + .. code-block:: python import mlflow @@ -94,6 +95,7 @@ In order to use the ``fluent`` API, we'll need to set the global reference to th address. We do this via the following command: .. code-section:: + .. code-block:: python mlflow.set_tracking_uri("http://127.0.0.1:8080") @@ -104,6 +106,7 @@ to log runs to. The parent-child relationship of Experiments to Runs and its uti clear once we start iterating over some ideas and need to compare the results of our tests. .. code-section:: + .. code-block:: python # Sets the current active experiment to the "Apple_Models" experiment and @@ -123,6 +126,7 @@ Firstly, let's look at what we're going to be running. Following the code displa an annotated version of the code. .. code-section:: + .. code-block:: python # Split the data into features and target and drop irrelevant date field and target field diff --git a/docs/source/getting-started/quickstart-1/index.rst b/docs/source/getting-started/quickstart-1/index.rst index e96a1f5fcaa6c..01386695e8227 100644 --- a/docs/source/getting-started/quickstart-1/index.rst +++ b/docs/source/getting-started/quickstart-1/index.rst @@ -86,6 +86,7 @@ In addition, or if you are using a library for which ``autolog`` is not yet supp This example demonstrates the use of these functions: .. code-section:: + .. code-block:: python import os diff --git a/docs/source/getting-started/quickstart-2/index.rst b/docs/source/getting-started/quickstart-2/index.rst index 7a7002ab719bd..8b419ce91848f 100644 --- a/docs/source/getting-started/quickstart-2/index.rst +++ b/docs/source/getting-started/quickstart-2/index.rst @@ -195,7 +195,7 @@ Choose **Chart view**. Choose the **Parallel coordinates** graph and configure i class="align-center" id="chart-view" alt="Screenshot of MLflow tracking UI parallel coordinates graph showing runs" - > + /> The red graphs on this graph are runs that fared poorly. The lowest one is a baseline run with both **lr** and **momentum** set to 0.0. That baseline run has an RMSE of ~0.89. The other red lines show that high **momentum** can also lead to poor results with this problem and architecture. diff --git a/docs/source/llms/custom-pyfunc-for-llms/index.rst b/docs/source/llms/custom-pyfunc-for-llms/index.rst index dbaa694acbe78..65de43cb7e620 100644 --- a/docs/source/llms/custom-pyfunc-for-llms/index.rst +++ b/docs/source/llms/custom-pyfunc-for-llms/index.rst @@ -27,7 +27,7 @@ Explore the Tutorial .. raw:: html - View the Custom Pyfunc for LLMs Tutorial
+ View the Custom Pyfunc for LLMs Tutorial
.. toctree:: :maxdepth: 1 diff --git a/docs/source/llms/custom-pyfunc-for-llms/notebooks/index.rst b/docs/source/llms/custom-pyfunc-for-llms/notebooks/index.rst index 3eb14978ee03b..13717fa8e229e 100644 --- a/docs/source/llms/custom-pyfunc-for-llms/notebooks/index.rst +++ b/docs/source/llms/custom-pyfunc-for-llms/notebooks/index.rst @@ -64,7 +64,7 @@ If you'd like to run a copy of the notebooks locally in your environment, you ca .. raw:: html - Download the LLM Custom Pyfunc notebook
+ Download the LLM Custom Pyfunc notebook
.. note:: To execute the notebooks, ensure you either have a local MLflow Tracking Server running or adjust the ``mlflow.set_tracking_uri()`` to point to an active MLflow Tracking Server instance. diff --git a/docs/source/llms/gateway/guides/step1-create-gateway.rst b/docs/source/llms/gateway/guides/step1-create-gateway.rst index 734fa5374fa36..c8b52447b470d 100644 --- a/docs/source/llms/gateway/guides/step1-create-gateway.rst +++ b/docs/source/llms/gateway/guides/step1-create-gateway.rst @@ -8,6 +8,7 @@ dependencies, including ``uvicorn`` and ``fastapi``. Note that direct dependenci unnecessary, as all supported providers are abstracted from the developer. .. code-section:: + .. code-block:: bash :name: install-gateway @@ -22,6 +23,7 @@ of leaking the token in code. The AI Gateway, when started, will read the value variable without any additional action required. .. code-section:: + .. code-block:: bash :name: token @@ -37,6 +39,7 @@ service restart is not required for changes to take effect and can instead be do configuration file that is defined at server start, permitting dynamic route creation without downtime of the service. .. code-section:: + .. code-block:: yaml :name: configure-gateway @@ -85,6 +88,7 @@ the URL: ``http://localhost:5000``. To modify these default settings, use the ``mlflow gateway --help`` command to view additional configuration options. .. code-section:: + .. code-block:: bash :name: start-gateway diff --git a/docs/source/llms/gateway/guides/step2-query-gateway.rst b/docs/source/llms/gateway/guides/step2-query-gateway.rst index 1e4204052ed2f..3226877192e3d 100644 --- a/docs/source/llms/gateway/guides/step2-query-gateway.rst +++ b/docs/source/llms/gateway/guides/step2-query-gateway.rst @@ -22,6 +22,7 @@ Setup First, import the necessary functions and define the gateway URI. .. code-section:: + .. code-block:: python :name: setup @@ -39,6 +40,7 @@ which is the string the Language Model (LLM) will respond to. The gateway also a various other parameters. For detailed information, please refer to the documentation. .. code-section:: + .. code-block:: python :name: completions @@ -72,6 +74,7 @@ takes a list of dictionaries formatted as follows: For further details, please consult the documentation. .. code-section:: + .. code-block:: python :name: chat @@ -103,6 +106,7 @@ string or a list of strings. The gateway then processes these strings and return respective numerical vectors. Let's proceed with an example... .. code-section:: + .. code-block:: python :name: embeddings diff --git a/docs/source/llms/gateway/index.rst b/docs/source/llms/gateway/index.rst index 55392bcfd2cb5..2e395ae0e46dc 100644 --- a/docs/source/llms/gateway/index.rst +++ b/docs/source/llms/gateway/index.rst @@ -42,7 +42,7 @@ as fast as possible, the guides below will be your best first stop. .. raw:: html - View the AI Gateway Getting Started Guide
+ View the AI Gateway Getting Started Guide
.. _gateway-quickstart: diff --git a/docs/source/llms/index.rst b/docs/source/llms/index.rst index 8a353126c5f7c..ba6754ef4ee14 100644 --- a/docs/source/llms/index.rst +++ b/docs/source/llms/index.rst @@ -67,47 +67,47 @@ configuration and management of your LLM serving needs, select the provider that @@ -246,25 +246,25 @@ Select the integration below to read the documentation on how to leverage MLflow
- HuggingFace Logo + HuggingFace Logo
- Sentence Transformers Logo + Sentence Transformers Logo
- LangChain Logo + LangChain Logo
- OpenAI Logo + OpenAI Logo
diff --git a/docs/source/llms/llm-evaluate/index.rst b/docs/source/llms/llm-evaluate/index.rst index 67463fe80d060..20f8033a988d7 100644 --- a/docs/source/llms/llm-evaluate/index.rst +++ b/docs/source/llms/llm-evaluate/index.rst @@ -23,7 +23,7 @@ functionality for LLMs, please navigate to the notebook collection below: .. raw:: html - View the Notebook Guides
+ View the Notebook Guides
Quickstart ---------- diff --git a/docs/source/llms/llm-evaluate/notebooks/index.rst b/docs/source/llms/llm-evaluate/notebooks/index.rst index 82c4af9f14a18..db43cfef8c989 100644 --- a/docs/source/llms/llm-evaluate/notebooks/index.rst +++ b/docs/source/llms/llm-evaluate/notebooks/index.rst @@ -21,13 +21,13 @@ If you would like a copy of this notebook to execute in your environment, downlo .. raw:: html - Download the notebook
+ Download the notebook
To follow along and see the sections of the notebook guide, click below: .. raw:: html - View the Notebook
+ View the Notebook
RAG Evaluation Notebook @@ -37,12 +37,12 @@ If you would like a copy of this notebook to execute in your environment, downlo .. raw:: html - Download the notebook
+ Download the notebook
To follow along and see the sections of the notebook guide, click below: .. raw:: html - View the Notebook
+ View the Notebook
diff --git a/docs/source/llms/rag/index.rst b/docs/source/llms/rag/index.rst index b40351ea0a6f4..f82d88bf4ba47 100644 --- a/docs/source/llms/rag/index.rst +++ b/docs/source/llms/rag/index.rst @@ -41,7 +41,7 @@ Explore the Tutorial .. raw:: html - View the RAG Question Generation Tutorial
+ View the RAG Question Generation Tutorial
.. toctree:: :maxdepth: 1 diff --git a/docs/source/llms/rag/notebooks/index.rst b/docs/source/llms/rag/notebooks/index.rst index 6737f769e7bc7..0fc5aa0a6c687 100644 --- a/docs/source/llms/rag/notebooks/index.rst +++ b/docs/source/llms/rag/notebooks/index.rst @@ -22,10 +22,10 @@ If you would like a copy of this notebook to execute in your environment, downlo .. raw:: html - Download the notebook
+ Download the notebook
To follow along and see the sections of the notebook guide, click below: .. raw:: html - View the Notebook
+ View the Notebook
diff --git a/docs/source/python_api/mlflow.metrics.rst b/docs/source/python_api/mlflow.metrics.rst index b0c962f3c1f73..98e1cbfddf147 100644 --- a/docs/source/python_api/mlflow.metrics.rst +++ b/docs/source/python_api/mlflow.metrics.rst @@ -147,19 +147,20 @@ Parameters: extra_metrics = [ mlflow.metrics.precision_at_k(5), mlflow.metrics.precision_at_k(6), - mlflow.metrics.recall_at_k(4), - mlflow.metrics.recall_at_k(5) + mlflow.metrics.recall_at_k(5), + mlflow.metrics.ndcg_at_k(5) ] ) NOTE: In the 2nd method, it is recommended to omit the ``model_type`` as well, or else ``precision@3`` and ``recall@3`` will be calculated in addition to ``precision@5``, - ``precision@6``, ``recall@4``, and ``recall@5``. + ``precision@6``, ``recall@5``, and ``ndcg_at_k@5``. .. autofunction:: mlflow.metrics.precision_at_k .. autofunction:: mlflow.metrics.recall_at_k +.. autofunction:: mlflow.metrics.ndcg_at_k Users create their own :py:class:`EvaluationMetric ` using the :py:func:`make_metric ` factory function @@ -169,7 +170,7 @@ Users create their own :py:class:`EvaluationMetric Download the Introduction notebook
- Download the Basic Pyfunc notebook
- Download the Predict Override notebook
+ Download the Introduction notebook
+ Download the Basic Pyfunc notebook
+ Download the Predict Override notebook
.. note:: In order to run the notebooks, please ensure that you either have a local MLflow Tracking Server started or modify the diff --git a/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/notebooks/index.rst b/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/notebooks/index.rst index ce6b31e16a563..e5e9d09fc5770 100644 --- a/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/notebooks/index.rst +++ b/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/notebooks/index.rst @@ -70,9 +70,9 @@ clicking the respective links to each notebook in this guide: .. raw:: html - Download the main notebook
- Download the Parent-Child Runs notebook
- Download the Plot Logging in MLflow notebook
+ Download the main notebook
+ Download the Parent-Child Runs notebook
+ Download the Plot Logging in MLflow notebook
.. note:: In order to run the notebooks, please ensure that you either have a local MLflow Tracking Server started or modify the diff --git a/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part1-child-runs.rst b/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part1-child-runs.rst index abdc887d36fe1..0b195d673e568 100644 --- a/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part1-child-runs.rst +++ b/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part1-child-runs.rst @@ -105,6 +105,7 @@ relatively performance amongst our iterative trials. If we were to use each iteration as its own MLflow run, our code might look something like this: .. code-section:: + .. code-block:: python import random @@ -162,6 +163,7 @@ What happens when we need to run this again with some slight modifications? Our code might change in-place with the values being tested: .. code-section:: + .. code-block:: python def log_run(run_name, test_no): @@ -204,6 +206,7 @@ Adapting for Parent and Child Runs The code below demonstrates these modifications to our original hyperparameter tuning example. .. code-section:: + .. code-block:: python import random @@ -276,6 +279,7 @@ The real benefit of this nested architecture becomes much more apparent when we with different conditions of hyperparameter selection criteria. .. code-section:: + .. code-block:: python # Execute modified hyperparameter tuning runs with custom parameter choices @@ -291,6 +295,7 @@ with different conditions of hyperparameter selection criteria. ... and even more runs ... .. code-section:: + .. code-block:: python param_1_values = ["b", "c"] diff --git a/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part2-logging-plots.rst b/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part2-logging-plots.rst index e9c2afb6fe41b..fff4293a53435 100644 --- a/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part2-logging-plots.rst +++ b/docs/source/traditional-ml/hyperparameter-tuning-with-child-runs/part2-logging-plots.rst @@ -105,6 +105,7 @@ materialized plots, which, if not regenerated after data modification, can lead and errors in data representation. .. code-section:: + .. code-block:: python def plot_box_weekend(df, style="seaborn", plot_size=(10, 8)): @@ -152,6 +153,7 @@ remains seamlessly compatible with MLflow, ensuring the same level of organizati with additional flexibility in plot access and usage. .. code-section:: + .. code-block:: python def plot_correlation_matrix_and_save( @@ -213,6 +215,7 @@ the more generic artifact writer (it supports any file type) ``mlflow.log_artifa .. code-section:: + .. code-block:: python mlflow.set_tracking_uri("http://127.0.0.1:8080") diff --git a/docs/source/traditional-ml/index.rst b/docs/source/traditional-ml/index.rst index 72ad577ef7047..bcfafa7b78a02 100644 --- a/docs/source/traditional-ml/index.rst +++ b/docs/source/traditional-ml/index.rst @@ -32,37 +32,37 @@ The officially supported integrations for traditional ML libraries include: diff --git a/mlflow/_promptlab.py b/mlflow/_promptlab.py index e176463ef439a..b5fee6163acb6 100644 --- a/mlflow/_promptlab.py +++ b/mlflow/_promptlab.py @@ -4,6 +4,7 @@ import yaml +from mlflow.exceptions import MlflowException from mlflow.version import VERSION as __version__ # noqa: F401 @@ -31,12 +32,45 @@ def predict(self, inputs: pd.DataFrame) -> List[str]: prompt = re.sub(r"\{\{\s*" + key + r"\s*\}\}", value, prompt) model_parameters_as_dict = {param.key: param.value for param in self.model_parameters} - result = query( - route=self.model_route, data={"prompt": prompt, **model_parameters_as_dict} + query_data = self._construct_query_data(prompt) + + response = query( + route=self.model_route, data={**query_data, **model_parameters_as_dict} ) - results.append(result["candidates"][0]["text"]) + results.append(self._parse_gateway_response(response)) + return results + def _construct_query_data(self, prompt): + from mlflow.gateway import get_route + + route_type = get_route(self.model_route).route_type + + if route_type == "llm/v1/completions": + return {"prompt": prompt} + elif route_type == "llm/v1/chat": + return {"messages": [{"content": prompt, "role": "user"}]} + else: + raise MlflowException( + "Error when constructing gateway query: " + f"Unsupported route type for _PromptlabModel: {route_type}" + ) + + def _parse_gateway_response(self, response): + from mlflow.gateway import get_route + + route_type = get_route(self.model_route).route_type + + if route_type == "llm/v1/completions": + return response["candidates"][0]["text"] + elif route_type == "llm/v1/chat": + return response["candidates"][0]["message"]["content"] + else: + raise MlflowException( + "Error when parsing gateway response: " + f"Unsupported route type for _PromptlabModel: {route_type}" + ) + def _load_pyfunc(path): from mlflow import pyfunc diff --git a/mlflow/gateway/config.py b/mlflow/gateway/config.py index a25fabfeb3ab6..7d67dcff2d39c 100644 --- a/mlflow/gateway/config.py +++ b/mlflow/gateway/config.py @@ -116,6 +116,8 @@ def validate_openai_api_key(cls, value): @classmethod def _validate_field_compatibility(cls, info: Dict[str, Any]): + if not isinstance(info, dict): + return info api_type = (info.get("openai_api_type") or OpenAIAPIType.OPENAI).lower() if api_type == OpenAIAPIType.OPENAI: if info.get("openai_deployment_name") is not None: diff --git a/mlflow/metrics/__init__.py b/mlflow/metrics/__init__.py index cb7e8b3143007..4f826ca6110ff 100644 --- a/mlflow/metrics/__init__.py +++ b/mlflow/metrics/__init__.py @@ -10,6 +10,7 @@ _mape_eval_fn, _max_error_eval_fn, _mse_eval_fn, + _ndcg_at_k_eval_fn, _precision_at_k_eval_fn, _precision_eval_fn, _r2_score_eval_fn, @@ -239,7 +240,7 @@ def precision_at_k(k) -> EvaluationMetric: This metric computes a score between 0 and 1 for each row representing the precision of the retriever model at the given ``k`` value. If no relevant documents are retrieved, the score is - 0, indicating that no relevant docs were retrieved. Let ``x = min(k, # of retrieved doc IDs)``. + 0, indicating that no relevant docs are retrieved. Let ``x = min(k, # of retrieved doc IDs)``. Then, in all other cases, the precision at k is calculated as follows: ``precision_at_k`` = (# of relevant retrieved doc IDs in top-``x`` ranked docs) / ``x``. @@ -258,8 +259,8 @@ def recall_at_k(k) -> EvaluationMetric: This metric computes a score between 0 and 1 for each row representing the recall ability of the retriever model at the given ``k`` value. If no ground truth doc IDs are provided and no - documents were retrieved, the score is 1. However, if no ground truth doc IDs are provided and - documents were retrieved, the score is 0. In all other cases, the recall at k is calculated as + documents are retrieved, the score is 1. However, if no ground truth doc IDs are provided and + documents are retrieved, the score is 0. In all other cases, the recall at k is calculated as follows: ``recall_at_k`` = (# of unique relevant retrieved doc IDs in top-``k`` ranked docs) / (# of @@ -272,6 +273,35 @@ def recall_at_k(k) -> EvaluationMetric: ) +@experimental +def ndcg_at_k(k) -> EvaluationMetric: + """ + This function will create a metric for evaluating `NDCG@k`_ for retriever models. + + NDCG score is capable of handling non-binary notions of relevance. However, for simplicity, + we use binary relevance here. The relevance score for documents in the ground truth is 1, + and the relevance score for documents not in the ground truth is 0. + + The NDCG score is calculated using sklearn.metrics.ndcg_score with the following edge cases + on top of the sklearn implementation: + + 1. If no ground truth doc IDs are provided and no documents are retrieved, the score is 1. + 2. If no ground truth doc IDs are provided and documents are retrieved, the score is 0. + 3. If ground truth doc IDs are provided and no documents are retrieved, the score is 0. + 4. If duplicate doc IDs are retrieved and the duplicate doc IDs are in the ground truth, + they will be treated as different docs. For example, if the ground truth doc IDs are + [1, 2] and the retrieved doc IDs are [1, 1, 1, 3], the score will be equavalent to + ground truth doc IDs [10, 11, 12, 2] and retrieved doc IDs [10, 11, 12, 3]. + + .. _NDCG@k: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.ndcg_score.html + """ + return make_metric( + eval_fn=_ndcg_at_k_eval_fn(k), + greater_is_better=True, + name=f"ndcg_at_{k}", + ) + + # General Regression Metrics def mae() -> EvaluationMetric: """ diff --git a/mlflow/metrics/genai/genai_metric.py b/mlflow/metrics/genai/genai_metric.py index 769ea5d114874..e6769539bac5f 100644 --- a/mlflow/metrics/genai/genai_metric.py +++ b/mlflow/metrics/genai/genai_metric.py @@ -262,16 +262,9 @@ def eval_fn( error_code=INVALID_PARAMETER_VALUE, ) - def score_model_on_one_payload( - indx, - input, - output, - grading_context_columns, - eval_values, - evaluation_context, - eval_parameters, - eval_model, - ): + # generate grading payloads + grading_payloads = [] + for indx, (input, output) in enumerate(zip(inputs, outputs)): try: arg_string = _format_args_string(grading_context_columns, eval_values, indx) except Exception as e: @@ -287,12 +280,19 @@ def score_model_on_one_payload( "parameter\n" "- input and output data are formatted correctly." ) - payload = { - "prompt": evaluation_context["eval_prompt"].format( - input=input, output=output, grading_context_columns=arg_string - ), - **eval_parameters, - } + grading_payloads.append( + { + "prompt": evaluation_context["eval_prompt"].format( + input=input, output=output, grading_context_columns=arg_string + ), + **eval_parameters, + } + ) + + def score_model_on_one_payload( + payload, + eval_model, + ): try: raw_result = model_utils.score_model_on_payload(eval_model, payload) return _extract_score_and_justification(raw_result) @@ -317,16 +317,10 @@ def score_model_on_one_payload( futures = { executor.submit( score_model_on_one_payload, - indx, - input, - output, - grading_context_columns, - eval_values, - evaluation_context, - eval_parameters, + payload, eval_model, ): indx - for indx, (input, output) in enumerate(zip(inputs, outputs)) + for indx, payload in enumerate(grading_payloads) } as_comp = as_completed(futures) diff --git a/mlflow/metrics/metric_definitions.py b/mlflow/metrics/metric_definitions.py index 5373af40f05a4..672f4d1859e9d 100644 --- a/mlflow/metrics/metric_definitions.py +++ b/mlflow/metrics/metric_definitions.py @@ -373,6 +373,100 @@ def _fn(predictions, targets): return _fn +def _expand_duplicate_retrieved_docs(predictions, targets): + counter = {} + expanded_predictions = [] + expanded_targets = targets + for doc_id in predictions: + if doc_id not in counter: + counter[doc_id] = 1 + expanded_predictions.append(doc_id) + else: + counter[doc_id] += 1 + new_doc_id = ( + f"{doc_id}_bc574ae_{counter[doc_id]}" # adding a random string to avoid collisions + ) + expanded_predictions.append(new_doc_id) + expanded_targets.add(new_doc_id) + return expanded_predictions, expanded_targets + + +def _prepare_row_for_ndcg(predictions, targets): + """Prepare data one row from predictions and targets to y_score, y_true for ndcg calculation. + + Args: + predictions: A list of strings of at most k doc IDs retrieved. + targets: A list of strings of ground-truth doc IDs. + + Returns: + y_true : ndarray of shape (1, n_docs) Representing the ground-truth relevant docs. + n_docs is the number of unique docs in union of predictions and targets. + y_score : ndarray of shape (1, n_docs) Representing the retrieved docs. + n_docs is the number of unique docs in union of predictions and targets. + """ + # sklearn does an internal sort of y_score, so to preserve the order of our retrieved + # docs, we need to modify the relevance value slightly + eps = 1e-6 + + # support predictions containing duplicate doc ID + targets = set(targets) + predictions, targets = _expand_duplicate_retrieved_docs(predictions, targets) + + all_docs = targets.union(predictions) + doc_id_to_index = {doc_id: i for i, doc_id in enumerate(all_docs)} + n_labels = max(len(doc_id_to_index), 2) # sklearn.metrics.ndcg_score requires at least 2 labels + y_true = np.zeros((1, n_labels), dtype=np.float32) + y_score = np.zeros((1, n_labels), dtype=np.float32) + for i, doc_id in enumerate(predictions): + # "1 - i * eps" means we assign higher score to docs that are ranked higher, + # but all scores are still approximately 1. + y_score[0, doc_id_to_index[doc_id]] = 1 - i * eps + for doc_id in targets: + y_true[0, doc_id_to_index[doc_id]] = 1 + return y_score, y_true + + +def _ndcg_at_k_eval_fn(k): + if not (isinstance(k, int) and k > 0): + _logger.warning( + f"Cannot calculate 'ndcg_at_k' for invalid parameter 'k'. " + f"'k' should be a positive integer; found: {k}. Skipping metric logging." + ) + return noop + + def _fn(predictions, targets): + from sklearn.metrics import ndcg_score + + if not _validate_list_str_data( + predictions, "ndcg_at_k", predictions_col_specifier + ) or not _validate_list_str_data(targets, "ndcg_at_k", targets_col_specifier): + return + + scores = [] + for ground_truth, retrieved in zip(targets, predictions): + # 1. If no ground truth doc IDs are provided and no documents are retrieved, + # the score is 1. + if len(retrieved) == 0 and len(ground_truth) == 0: + scores.append(1) # no error is made + continue + # 2. If no ground truth doc IDs are provided and documents are retrieved, + # the score is 0. + # 3. If ground truth doc IDs are provided and no documents are retrieved, + # the score is 0. + if len(retrieved) == 0 or len(ground_truth) == 0: + scores.append(0) + continue + + # only include the top k retrieved chunks + y_score, y_true = _prepare_row_for_ndcg(retrieved[:k], ground_truth) + score = ndcg_score(y_true, y_score, k=len(retrieved[:k]), ignore_ties=True) + scores.append(score) + + return MetricValue(scores=scores, aggregate_results=standard_aggregations(scores)) + + return _fn + + def _recall_at_k_eval_fn(k): if not (isinstance(k, int) and k > 0): _logger.warning( diff --git a/mlflow/models/evaluation/base.py b/mlflow/models/evaluation/base.py index ae50027f9022b..4648b5ab9ccf4 100644 --- a/mlflow/models/evaluation/base.py +++ b/mlflow/models/evaluation/base.py @@ -1313,8 +1313,9 @@ def evaluate( https://pypi.org/project/textstat - For retriever models, the default evaluator logs: - - **metrics**: :mod:`precision_at_k(k) ` and - :mod:`recall_at_k(k) ` - both have a default value of + - **metrics**: :mod:`precision_at_k(k) `, + :mod:`recall_at_k(k) ` and + :mod:`ndcg_at_k(k) ` - all have a default value of ``retriever_k`` = 3. - **artifacts**: A JSON file containing the inputs, outputs, targets, and per-row metrics of the model in tabular format. @@ -1364,8 +1365,9 @@ def evaluate( predictions to column names used when invoking the evaluation functions. - **retriever_k**: A parameter used when ``model_type="retriever"`` as the number of top-ranked retrieved documents to use when computing the built-in metric - :mod:`precision_at_k(k) ` and - :mod:`recall_at_k(k) `. Default value is 3. For all other + :mod:`precision_at_k(k) `, + :mod:`recall_at_k(k) ` and + :mod:`ndcg_at_k(k) `. Default value is 3. For all other model types, this parameter will be ignored. - Limitations of evaluation dataset: diff --git a/mlflow/models/evaluation/default_evaluator.py b/mlflow/models/evaluation/default_evaluator.py index 1ab20e50ff5fb..a07022f023d49 100644 --- a/mlflow/models/evaluation/default_evaluator.py +++ b/mlflow/models/evaluation/default_evaluator.py @@ -31,6 +31,7 @@ ari_grade_level, exact_match, flesch_kincaid_grade_level, + ndcg_at_k, precision_at_k, recall_at_k, rouge1, @@ -1714,6 +1715,7 @@ def _evaluate( self.builtin_metrics = [ precision_at_k(retriever_k), recall_at_k(retriever_k), + ndcg_at_k(retriever_k), ] eval_df = pd.DataFrame({"prediction": copy.deepcopy(self.y_pred)}) diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index d88294f576cd8..76ea903cc2ac9 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -96,6 +96,7 @@ from mlflow.tracking.registry import UnsupportedModelRegistryStoreURIException from mlflow.utils.file_utils import local_file_uri_to_path from mlflow.utils.mime_type_utils import _guess_mime_type +from mlflow.utils.os import is_windows from mlflow.utils.promptlab_utils import _create_promptlab_run_impl from mlflow.utils.proto_json_utils import message_to_json, parse_dict from mlflow.utils.string_utils import is_string_type @@ -552,6 +553,7 @@ def validate_path_is_safe(path): or ".." in path.split("/") or pathlib.PureWindowsPath(path).is_absolute() or pathlib.PurePosixPath(path).is_absolute() + or (is_windows() and len(path) >= 2 and path[1] == ":") ): raise MlflowException(f"Invalid path: {path}", error_code=INVALID_PARAMETER_VALUE) diff --git a/mlflow/store/artifact/models_artifact_repo.py b/mlflow/store/artifact/models_artifact_repo.py index 1ef8f91fcdbff..32149cfb00690 100644 --- a/mlflow/store/artifact/models_artifact_repo.py +++ b/mlflow/store/artifact/models_artifact_repo.py @@ -1,3 +1,5 @@ +import logging +import os import urllib.parse import mlflow @@ -20,6 +22,8 @@ REGISTERED_MODEL_META_FILE_NAME = "registered_model_meta" +_logger = logging.getLogger(__name__) + class ModelsArtifactRepository(ArtifactRepository): """ @@ -169,8 +173,14 @@ def download_artifacts(self, artifact_path, dst_path=None): :return: Absolute path of the local filesystem location containing the desired artifacts. """ + from mlflow.models.model import MLMODEL_FILE_NAME + model_path = self.repo.download_artifacts(artifact_path, dst_path) - self._add_registered_model_meta_file(model_path) + # NB: only add the registered model metadata iff the artifact path is at the root model + # directory. For individual files or subdirectories within the model directory, do not + # create the metadata file. + if os.path.isdir(model_path) and MLMODEL_FILE_NAME in os.listdir(model_path): + self._add_registered_model_meta_file(model_path) return model_path diff --git a/mlflow/system_metrics/system_metrics_monitor.py b/mlflow/system_metrics/system_metrics_monitor.py index 16b1d0ca2cc34..09d0bd6059fa7 100644 --- a/mlflow/system_metrics/system_metrics_monitor.py +++ b/mlflow/system_metrics/system_metrics_monitor.py @@ -7,6 +7,7 @@ MLFLOW_SYSTEM_METRICS_SAMPLES_BEFORE_LOGGING, MLFLOW_SYSTEM_METRICS_SAMPLING_INTERVAL, ) +from mlflow.exceptions import MlflowException from mlflow.system_metrics.metrics.cpu_monitor import CPUMonitor from mlflow.system_metrics.metrics.disk_monitor import DiskMonitor from mlflow.system_metrics.metrics.gpu_monitor import GPUMonitor @@ -36,9 +37,18 @@ class SystemMetricsMonitor: samples_before_logging: int, default to 1. The number of samples to aggregate before logging. Will be overridden by `MLFLOW_SYSTEM_METRICS_SAMPLES_BEFORE_LOGGING` evnironment variable. + resume_logging: bool, default to False. If True, we will resume the system metrics logging + from the `run_id`, and the first step to log will be the last step of `run_id` + 1, if + False, system metrics logging will start from step 0. """ - def __init__(self, run_id, sampling_interval=10, samples_before_logging=1): + def __init__( + self, + run_id, + sampling_interval=10, + samples_before_logging=1, + resume_logging=False, + ): from mlflow.utils.autologging_utils import BatchMetricsLogger # Instantiate default monitors. @@ -60,8 +70,26 @@ def __init__(self, run_id, sampling_interval=10, samples_before_logging=1): self.mlflow_logger = BatchMetricsLogger(self._run_id) self._shutdown_event = threading.Event() self._process = None - self._logging_step = 0 self._metrics_prefix = "system/" + self._logging_step = self._get_next_logging_step(run_id) if resume_logging else 0 + + def _get_next_logging_step(self, run_id): + from mlflow.tracking.client import MlflowClient + + client = MlflowClient() + try: + run = client.get_run(run_id) + except MlflowException: + return 0 + system_metric_name = None + for metric_name in run.data.metrics.keys(): + if metric_name.startswith(self._metrics_prefix): + system_metric_name = metric_name + break + if system_metric_name is None: + return 0 + metric_history = client.get_metric_history(run_id, system_metric_name) + return metric_history[-1].step + 1 def start(self): """Start monitoring system metrics.""" diff --git a/mlflow/tracking/fluent.py b/mlflow/tracking/fluent.py index 8e2029f41a1b7..5c56421e76ac3 100644 --- a/mlflow/tracking/fluent.py +++ b/mlflow/tracking/fluent.py @@ -376,32 +376,32 @@ def start_run( tags=resolved_tags, run_name=run_name, ) - if log_system_metrics is None: - # if `log_system_metrics` is not specified, we will check environment variable. - log_system_metrics = MLFLOW_ENABLE_SYSTEM_METRICS_LOGGING.get() - if log_system_metrics: - # Ensure psutil is installed. It was moved to an optional dependency, as it doesn't - # have binary for Arm64 Linux and requires build from CPython which is a headache. - # https://github.com/giampaolo/psutil/issues/1972 - if importlib.util.find_spec("psutil") is None: - raise MlflowException( - "Failed to start system metrics monitoring as package `psutil` is not " - "installed. Run `pip install psutil` to resolve the issue, " - "otherwise you can disable system metrics logging by passing " - "`log_system_metrics=False` to `start_run()` or setting environment " - f"variable {MLFLOW_ENABLE_SYSTEM_METRICS_LOGGING} to False." - ) - try: - from mlflow.system_metrics.system_metrics_monitor import SystemMetricsMonitor + if log_system_metrics is None: + # If `log_system_metrics` is not specified, we will check environment variable. + log_system_metrics = MLFLOW_ENABLE_SYSTEM_METRICS_LOGGING.get() + + if log_system_metrics: + if importlib.util.find_spec("psutil") is None: + raise MlflowException( + "Failed to start system metrics monitoring as package `psutil` is not installed. " + "Please run `pip install psutil` to resolve the issue, otherwise you can disable " + "system metrics logging by passing `log_system_metrics=False` to " + "`mlflow.start_run()` or calling `mlflow.disable_system_metrics_logging`." + ) + try: + from mlflow.system_metrics.system_metrics_monitor import SystemMetricsMonitor - system_monitor = SystemMetricsMonitor(active_run_obj.info.run_id) - global run_id_to_system_metrics_monitor + system_monitor = SystemMetricsMonitor( + active_run_obj.info.run_id, + resume_logging=existing_run_id is not None, + ) + global run_id_to_system_metrics_monitor - run_id_to_system_metrics_monitor[active_run_obj.info.run_id] = system_monitor - system_monitor.start() - except Exception as e: - _logger.error(f"Failed to start system metrics monitoring: {e}.") + run_id_to_system_metrics_monitor[active_run_obj.info.run_id] = system_monitor + system_monitor.start() + except Exception as e: + _logger.error(f"Failed to start system metrics monitoring: {e}.") _active_run_stack.append(ActiveRun(active_run_obj)) return _active_run_stack[-1] diff --git a/mlflow/utils/async_logging/async_logging_queue.py b/mlflow/utils/async_logging/async_logging_queue.py index 77579b75db853..7878d92df044b 100644 --- a/mlflow/utils/async_logging/async_logging_queue.py +++ b/mlflow/utils/async_logging/async_logging_queue.py @@ -47,9 +47,8 @@ def _at_exit_callback(self) -> None: try: # Stop the data processing thread self._stop_data_logging_thread_event.set() - # Waits till queue is drained. - self._run_data_logging_thread.result() - self._batch_logging_threadpool.shutdown(wait=False) + # Waits till logging queue is drained. + self._batch_logging_thread.join() self._batch_status_check_threadpool.shutdown(wait=False) except Exception as e: _logger.error(f"Encountered error while trying to finish logging: {e}") @@ -132,14 +131,10 @@ def __getstate__(self): del state["_run_data_logging_thread"] if "_stop_data_logging_thread_event" in state: del state["_stop_data_logging_thread_event"] - if "_batch_logging_threadpool" in state: - del state["_batch_logging_threadpool"] + if "_batch_logging_thread" in state: + del state["_batch_logging_thread"] if "_batch_status_check_threadpool" in state: del state["_batch_status_check_threadpool"] - if "_run_data_logging_thread" in state: - del state["_run_data_logging_thread"] - if "_stop_data_logging_thread_event" in state: - del state["_stop_data_logging_thread_event"] return state @@ -158,7 +153,7 @@ def __setstate__(self, state): self._queue = Queue() self._lock = threading.RLock() self._is_activated = False - self._batch_logging_threadpool = None + self._batch_logging_thread = None self._batch_status_check_threadpool = None self._stop_data_logging_thread_event = None @@ -193,7 +188,6 @@ def log_batch_async( ) self._queue.put(batch) - operation_future = self._batch_status_check_threadpool.submit(self._wait_for_batch, batch) return RunOperations(operation_futures=[operation_future]) @@ -217,14 +211,17 @@ def activate(self) -> None: self._stop_data_logging_thread_event = threading.Event() # Keeping max_workers=1 so that there are no two threads - self._batch_logging_threadpool = ThreadPoolExecutor(max_workers=1) - - self._batch_status_check_threadpool = ThreadPoolExecutor(max_workers=10) - - self._run_data_logging_thread = self._batch_logging_threadpool.submit( - self._logging_loop - ) # concurrent.futures.Future[self._logging_loop] + self._batch_logging_thread = threading.Thread( + target=self._logging_loop, + name="MLflowAsyncLoggingLoop", + daemon=True, + ) + self._batch_status_check_threadpool = ThreadPoolExecutor( + max_workers=10, + thread_name_prefix="MLflowAsyncLoggingStatusCheck", + ) + self._batch_logging_thread.start() atexit.register(self._at_exit_callback) self._is_activated = True diff --git a/mlflow/utils/credentials.py b/mlflow/utils/credentials.py index d7d9bc0c83827..0f2099894ab8f 100644 --- a/mlflow/utils/credentials.py +++ b/mlflow/utils/credentials.py @@ -90,7 +90,7 @@ def _validate_databricks_auth(): # If the credential is invalid, the command will return non-zero exit code. # If both host and credential are valid, it will return zero exit code. result = subprocess.run( - ["databricks", "tokens", "list"], + ["databricks", "clusters", "list-zones"], timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/requirements/doc-min-requirements.txt b/requirements/doc-min-requirements.txt index e73f9d360fa18..c825dfc2d7678 100644 --- a/requirements/doc-min-requirements.txt +++ b/requirements/doc-min-requirements.txt @@ -5,4 +5,4 @@ jinja2==3.0.3 # to be compatible with jinja2==3.0.3 flask<=2.2.5 sphinx-autobuild -sphinx-click +sphinx-click \ No newline at end of file diff --git a/tests/evaluate/test_default_evaluator.py b/tests/evaluate/test_default_evaluator.py index b57db8b285add..144143780b52c 100644 --- a/tests/evaluate/test_default_evaluator.py +++ b/tests/evaluate/test_default_evaluator.py @@ -3222,6 +3222,7 @@ def validate_retriever_logged_data(logged_data, k=3): "retrieved_context", f"precision_at_{k}/score", f"recall_at_{k}/score", + f"ndcg_at_{k}/score", "ground_truth", } @@ -3231,6 +3232,7 @@ def validate_retriever_logged_data(logged_data, k=3): assert logged_data["retrieved_context"].tolist() == [["doc1", "doc3", "doc2"]] * 3 assert (logged_data[f"precision_at_{k}/score"] <= 1).all() assert (logged_data[f"recall_at_{k}/score"] <= 1).all() + assert (logged_data[f"ndcg_at_{k}/score"] <= 1).all() assert logged_data["ground_truth"].tolist() == [["doc1", "doc2"]] * 3 @@ -3259,6 +3261,9 @@ def fn(X): "recall_at_3/mean": 1.0, "recall_at_3/p90": 1.0, "recall_at_3/variance": 0.0, + "ndcg_at_3/mean": pytest.approx(0.9197207891481877), + "ndcg_at_3/p90": pytest.approx(0.9197207891481877), + "ndcg_at_3/variance": 0.0, } client = mlflow.MlflowClient() artifacts = [a.path for a in client.list_artifacts(run.info.run_id)] @@ -3286,6 +3291,9 @@ def fn(X): "recall_at_6/mean": 1.0, "recall_at_6/p90": 1.0, "recall_at_6/variance": 0.0, + "ndcg_at_6/mean": pytest.approx(0.9197207891481877), + "ndcg_at_6/p90": pytest.approx(0.9197207891481877), + "ndcg_at_6/variance": 0.0, } # test with default k @@ -3304,6 +3312,9 @@ def fn(X): "recall_at_3/mean": 1.0, "recall_at_3/p90": 1.0, "recall_at_3/variance": 0.0, + "ndcg_at_3/mean": pytest.approx(0.9197207891481877), + "ndcg_at_3/p90": pytest.approx(0.9197207891481877), + "ndcg_at_3/variance": 0.0, } # test with multiple chunks from same doc @@ -3332,6 +3343,9 @@ def fn2(X): "recall_at_3/mean": 1.0, "recall_at_3/p90": 1.0, "recall_at_3/variance": 0.0, + "ndcg_at_3/mean": 1.0, + "ndcg_at_3/p90": 1.0, + "ndcg_at_3/variance": 0.0, } # test with empty retrieved doc @@ -3358,6 +3372,9 @@ def fn3(X): "recall_at_4/mean": 0, "recall_at_4/p90": 0, "recall_at_4/variance": 0, + "ndcg_at_4/mean": 0.0, + "ndcg_at_4/p90": 0.0, + "ndcg_at_4/variance": 0.0, } # test with a static dataset @@ -3384,6 +3401,9 @@ def fn3(X): "recall_at_3/mean": 0.5, "recall_at_3/p90": 0.5, "recall_at_3/variance": 0.0, + "ndcg_at_3/mean": pytest.approx(0.6131471927654585), + "ndcg_at_3/p90": pytest.approx(0.6131471927654585), + "ndcg_at_3/variance": 0.0, "precision_at_4/mean": 1 / 3, "precision_at_4/p90": 1 / 3, "precision_at_4/variance": 0.0, @@ -3409,6 +3429,9 @@ def fn3(X): "recall_at_3/mean": 0.5, "recall_at_3/p90": 0.5, "recall_at_3/variance": 0.0, + "ndcg_at_3/mean": pytest.approx(0.6131471927654585), + "ndcg_at_3/p90": pytest.approx(0.6131471927654585), + "ndcg_at_3/variance": 0.0, } # silent failure with evaluator_config method too! @@ -3438,7 +3461,11 @@ def fn(X): model=fn, data=X, targets="ground_truth", - extra_metrics=[mlflow.metrics.precision_at_k(4), mlflow.metrics.recall_at_k(4)], + extra_metrics=[ + mlflow.metrics.precision_at_k(4), + mlflow.metrics.recall_at_k(4), + mlflow.metrics.ndcg_at_k(4), + ], ) run = mlflow.get_run(run.info.run_id) assert ( @@ -3451,6 +3478,9 @@ def fn(X): "recall_at_4/mean": 1.0, "recall_at_4/p90": 1.0, "recall_at_4/variance": 0.0, + "ndcg_at_4/mean": pytest.approx(0.9197207891481877), + "ndcg_at_4/p90": pytest.approx(0.9197207891481877), + "ndcg_at_4/variance": 0.0, } ) client = mlflow.MlflowClient() diff --git a/tests/metrics/test_metric_definitions.py b/tests/metrics/test_metric_definitions.py index d75fa2a8a2eef..634afb7fd3862 100644 --- a/tests/metrics/test_metric_definitions.py +++ b/tests/metrics/test_metric_definitions.py @@ -13,6 +13,7 @@ mape, max_error, mse, + ndcg_at_k, precision_at_k, precision_score, r2_score, @@ -266,3 +267,69 @@ def test_recall_at_k(): "p90": 0.8, "variance": 0.1, } + + +def test_ndcg_at_k(): + # normal cases + data = pd.DataFrame( + [ + {"target": [], "prediction": [], "k": [3], "ndcg": 1}, # no error is made + {"target": [], "prediction": ["1", "2"], "k": [3], "ndcg": 0}, + {"target": ["1"], "prediction": [], "k": [3], "ndcg": 0}, + {"target": ["1"], "prediction": ["1"], "k": [3], "ndcg": 1}, + {"target": ["1"], "prediction": ["2"], "k": [3], "ndcg": 0}, + ] + ) + predictions = data["prediction"] + targets = data["target"] + result = ndcg_at_k(3).eval_fn(predictions, targets) + + assert result.scores == data["ndcg"].to_list() + assert pytest.approx(result.aggregate_results["mean"]) == 0.4 + assert pytest.approx(result.aggregate_results["p90"]) == 1.0 + assert pytest.approx(result.aggregate_results["variance"]) == 0.24 + + # test different k values + predictions = pd.Series([["1", "2"]]) + targets = pd.Series([["1"]]) + ndcg = [1, 1, 1] + for i in range(3): + k = i + 1 + result = ndcg_at_k(k).eval_fn(predictions, targets) + assert pytest.approx(result.scores[0]) == ndcg[i] + + # test different k values and prediction orders + predictions = pd.Series([["2", "1", "3"]]) + targets = pd.Series([["1", "2", "3"]]) + ndcg = [1, 1, 1, 1] + for i in range(4): + k = i + 1 + result = ndcg_at_k(k).eval_fn(predictions, targets) + assert pytest.approx(result.scores[0]) == ndcg[i] + + # test different k values + predictions = pd.Series([["4", "5", "1"]]) + targets = pd.Series([["1", "2", "3"]]) + ndcg = [0, 0, 0.2346394, 0.2346394] + for i in range(4): + k = i + 1 + result = ndcg_at_k(k).eval_fn(predictions, targets) + assert pytest.approx(result.scores[0]) == ndcg[i] + + # test duplicate predictions + predictions = pd.Series([["1", "1", "2", "5", "5"], ["1_1", "1_2", "2", "5", "6"]]) + targets = pd.Series([["1", "2", "3"], ["1_1", "1_2", "2", "3"]]) + for i in range(4): + k = i + 1 + result = ndcg_at_k(k).eval_fn(predictions, targets) + # row 1 and 2 have the same ndcg score + assert pytest.approx(result.scores[0]) == pytest.approx(result.scores[1]) + + # test duplicate targets + predictions = pd.Series([["1", "2", "3"], ["1", "2", "3"]]) + targets = pd.Series([["1", "1", "1"], ["1"]]) + for i in range(4): + k = i + 1 + result = ndcg_at_k(k).eval_fn(predictions, targets) + # row 1 and 2 have the same ndcg score + assert pytest.approx(result.scores[0]) == pytest.approx(result.scores[1]) diff --git a/tests/promptlab/test_promptlab_model.py b/tests/promptlab/test_promptlab_model.py index f152852a94b15..bd25a0768eb5a 100644 --- a/tests/promptlab/test_promptlab_model.py +++ b/tests/promptlab/test_promptlab_model.py @@ -4,6 +4,18 @@ from mlflow._promptlab import _PromptlabModel from mlflow.entities.param import Param +from mlflow.gateway import set_gateway_uri + +set_gateway_uri("http://localhost:5000") + + +def construct_model(route): + return _PromptlabModel( + "Write me a story about {{ thing }}.", + [Param(key="thing", value="books")], + [Param(key="temperature", value=0.5), Param(key="max_tokens", value=10)], + route, + ) def test_promptlab_prompt_replacement(): @@ -15,13 +27,12 @@ def test_promptlab_prompt_replacement(): ] ) - prompt_parameters = [Param(key="thing", value="books")] - model_parameters = [Param(key="temperature", value=0.5), Param(key="max_tokens", value=10)] - prompt_template = "Write me a story about {{ thing }}." - model_route = "completions" + model = construct_model("completions") + get_route_patch = mock.patch( + "mlflow.gateway.get_route", return_value=mock.Mock(route_type="llm/v1/completions") + ) - model = _PromptlabModel(prompt_template, prompt_parameters, model_parameters, model_route) - with mock.patch("mlflow.gateway.query") as mock_query: + with get_route_patch, mock.patch("mlflow.gateway.query") as mock_query: model.predict(data) calls = [ @@ -37,3 +48,41 @@ def test_promptlab_prompt_replacement(): ] mock_query.assert_has_calls(calls, any_order=True) + + +def test_promptlab_works_with_chat_route(): + mock_response = { + "candidates": [ + {"message": {"role": "user", "content": "test"}, "metadata": {"finish_reason": "stop"}} + ] + } + model = construct_model("chat") + get_route_patch = mock.patch( + "mlflow.gateway.get_route", + return_value=mock.Mock(route_type="llm/v1/chat"), + ) + + with get_route_patch, mock.patch("mlflow.gateway.query", return_value=mock_response): + response = model.predict(pd.DataFrame(data=[{"thing": "books"}])) + + assert response == ["test"] + + +def test_promptlab_works_with_completions_route(): + mock_response = { + "candidates": [ + { + "text": "test", + "metadata": {"finish_reason": "stop"}, + } + ] + } + model = construct_model("completions") + get_route_patch = mock.patch( + "mlflow.gateway.get_route", return_value=mock.Mock(route_type="llm/v1/completions") + ) + + with get_route_patch, mock.patch("mlflow.gateway.query", return_value=mock_response): + response = model.predict(pd.DataFrame(data=[{"thing": "books"}])) + + assert response == ["test"] diff --git a/tests/sklearn/test_sklearn_model_export.py b/tests/sklearn/test_sklearn_model_export.py index 9e15cbebb00eb..2b2f151cc52d6 100644 --- a/tests/sklearn/test_sklearn_model_export.py +++ b/tests/sklearn/test_sklearn_model_export.py @@ -28,6 +28,7 @@ from mlflow.models.utils import _read_example from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE, ErrorCode from mlflow.store._unity_catalog.registry.rest_store import UcModelRegistryStore +from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository from mlflow.store.artifact.s3_artifact_repo import S3ArtifactRepository from mlflow.tracking.artifact_utils import _download_artifact_from_uri from mlflow.types import DataType @@ -837,3 +838,24 @@ def test_model_size_bytes(sklearn_logreg_model, tmp_path): mlmodel = yaml.safe_load(tmp_path.joinpath("MLmodel").read_bytes()) assert mlmodel["model_size_bytes"] == expected_size + + +def test_model_registration_metadata_handling(sklearn_knn_model, tmp_path): + artifact_path = "model" + with mlflow.start_run(): + mlflow.sklearn.log_model( + sk_model=sklearn_knn_model.model, + artifact_path=artifact_path, + registered_model_name="test", + ) + model_uri = "models:/test/1" + + artifact_repository = get_artifact_repository(model_uri) + + dst_full = tmp_path.joinpath("full") + dst_full.mkdir() + + artifact_repository.download_artifacts("MLmodel", dst_full) + # This validates that the models artifact repo will not attempt to create a + # "registered model metadata" file if the source of an artifact download is a file. + assert os.listdir(dst_full) == ["MLmodel"] diff --git a/tests/store/artifact/test_models_artifact_repo.py b/tests/store/artifact/test_models_artifact_repo.py index 2998e537e15ab..240c890858eb4 100644 --- a/tests/store/artifact/test_models_artifact_repo.py +++ b/tests/store/artifact/test_models_artifact_repo.py @@ -1,5 +1,4 @@ from unittest import mock -from unittest.mock import Mock import pytest @@ -10,6 +9,7 @@ from mlflow.store.artifact.unity_catalog_models_artifact_repo import ( UnityCatalogModelsArtifactRepository, ) +from mlflow.utils.os import is_windows from tests.store.artifact.constants import ( UC_MODELS_ARTIFACT_REPOSITORY, @@ -111,50 +111,104 @@ def test_models_artifact_repo_init_with_stage_uri_and_not_using_databricks_regis get_repo_mock.assert_called_once_with(artifact_location) -def test_models_artifact_repo_uses_repo_download_artifacts(): +def test_models_artifact_repo_uses_repo_download_artifacts(tmp_path): """ - ``ModelsArtifactRepository`` should delegate `download_artifacts` to its - ``self.repo.download_artifacts`` function. + `ModelsArtifactRepository` should delegate `download_artifacts` to its + `self.repo.download_artifacts` function. """ artifact_location = "s3://blah_bucket/" + dummy_file = tmp_path / "dummy_file.txt" + dummy_file.touch() + with mock.patch.object( MlflowClient, "get_model_version_download_uri", return_value=artifact_location ), mock.patch.object(ModelsArtifactRepository, "_add_registered_model_meta_file"): model_uri = "models:/MyModel/12" models_repo = ModelsArtifactRepository(model_uri) - models_repo.repo = Mock() - models_repo.download_artifacts("artifact_path", "dst_path") - models_repo.repo.download_artifacts.assert_called_once() + models_repo.repo = mock.Mock(**{"download_artifacts.return_value": str(dummy_file)}) + + models_repo.download_artifacts("artifact_path", str(tmp_path)) + + models_repo.repo.download_artifacts.assert_called_once_with("artifact_path", str(tmp_path)) + +@pytest.mark.skipif(is_windows(), reason="This test fails on Windows") +def test_models_artifact_repo_download_with_real_files(tmp_path): + # Simulate an artifact repository + temp_remote_storage = tmp_path / "remote_storage" + model_dir = temp_remote_storage / "model_dir" + model_dir.mkdir(parents=True) + mlmodel_path = model_dir / "MLmodel" + mlmodel_path.touch() + + # Mock get_model_version_download_uri to return the path to the temp_remote_storage location + with mock.patch.object( + MlflowClient, "get_model_version_download_uri", return_value=str(model_dir) + ): + # Create ModelsArtifactRepository instance + models_repo = ModelsArtifactRepository("models:/MyModel/1") -def test_models_artifact_repo_add_registered_model_meta_file(): - from mlflow.store.artifact.models_artifact_repo import REGISTERED_MODEL_META_FILE_NAME + # Use another temporary directory as the download destination + temp_local_storage = tmp_path / "local_storage" + temp_local_storage.mkdir() - artifact_path = "artifact_path" - dst_path = "dst_path" + # Download artifacts + models_repo.download_artifacts("", str(temp_local_storage)) + + # Check if the files are downloaded correctly + downloaded_mlmodel_path = temp_local_storage / "MLmodel" + assert downloaded_mlmodel_path.exists() + + # Check if the metadata file is created + metadata_file_path = temp_local_storage / "registered_model_meta" + assert metadata_file_path.exists() + + +def test_models_artifact_repo_does_not_add_meta_for_file(tmp_path): + artifact_path = "artifact_file.txt" + model_name = "MyModel" + model_version = "12" artifact_location = f"s3://blah_bucket/{artifact_path}" - artifact_dst_path = f"{dst_path}/{artifact_path}" + + dummy_file = tmp_path / artifact_path + dummy_file.touch() + + with mock.patch.object( + MlflowClient, "get_model_version_download_uri", return_value=artifact_location + ), mock.patch.object( + ModelsArtifactRepository, "_add_registered_model_meta_file" + ) as add_meta_mock: + models_repo = ModelsArtifactRepository(f"models:/{model_name}/{model_version}") + models_repo.repo = mock.Mock(**{"download_artifacts.return_value": str(dummy_file)}) + + models_repo.download_artifacts(artifact_path, str(tmp_path)) + + add_meta_mock.assert_not_called() + + +def test_models_artifact_repo_does_not_add_meta_for_directory_without_mlmodel(tmp_path): + artifact_path = "artifact_directory" model_name = "MyModel" model_version = "12" + artifact_location = f"s3://blah_bucket/{artifact_path}" + + # Create a directory without an MLmodel file + dummy_dir = tmp_path / artifact_path + dummy_dir.mkdir() + dummy_file = dummy_dir / "dummy_file.txt" + dummy_file.touch() with mock.patch.object( MlflowClient, "get_model_version_download_uri", return_value=artifact_location - ), mock.patch("mlflow.store.artifact.models_artifact_repo.write_yaml") as write_yaml_mock: + ), mock.patch.object( + ModelsArtifactRepository, "_add_registered_model_meta_file" + ) as add_meta_mock: models_repo = ModelsArtifactRepository(f"models:/{model_name}/{model_version}") - models_repo.repo = Mock(**{"download_artifacts.return_value": artifact_dst_path}) - - models_repo.download_artifacts(artifact_path, dst_path) - - write_yaml_mock.assert_called_with( - artifact_dst_path, - REGISTERED_MODEL_META_FILE_NAME, - { - "model_name": model_name, - "model_version": model_version, - }, - overwrite=True, - ensure_yaml_extension=False, - ) + models_repo.repo = mock.Mock(**{"download_artifacts.return_value": str(dummy_dir)}) + + models_repo.download_artifacts(artifact_path, str(tmp_path)) + + add_meta_mock.assert_not_called() def test_split_models_uri(): diff --git a/tests/system_metrics/test_system_metrics_logging.py b/tests/system_metrics/test_system_metrics_logging.py index dc264b0c8088a..2a30b675fbc41 100644 --- a/tests/system_metrics/test_system_metrics_logging.py +++ b/tests/system_metrics/test_system_metrics_logging.py @@ -1,10 +1,21 @@ import threading import time +import pytest + import mlflow from mlflow.system_metrics.system_metrics_monitor import SystemMetricsMonitor +@pytest.fixture(autouse=True) +def disable_system_metrics_logging(): + yield + # Unset the environment variables to avoid affecting other test cases. + mlflow.disable_system_metrics_logging() + mlflow.set_system_metrics_sampling_interval(None) + mlflow.set_system_metrics_samples_before_logging(None) + + def test_manual_system_metrics_monitor(): with mlflow.start_run(log_system_metrics=False) as run: system_monitor = SystemMetricsMonitor( @@ -41,7 +52,7 @@ def test_manual_system_metrics_monitor(): assert name in metrics # Check the step is correctly logged. - metrics_history = mlflow.tracking.MlflowClient().get_metric_history( + metrics_history = mlflow.MlflowClient().get_metric_history( run.info.run_id, "system/cpu_utilization_percentage" ) assert metrics_history[-1].step > 0 @@ -82,12 +93,51 @@ def test_automatic_system_metrics_monitor(): assert name in metrics # Check the step is correctly logged. - metrics_history = mlflow.tracking.MlflowClient().get_metric_history( + metrics_history = mlflow.MlflowClient().get_metric_history( run.info.run_id, "system/cpu_utilization_percentage" ) assert metrics_history[-1].step > 0 - # Unset the environment variables to avoid affecting other test cases. - mlflow.disable_system_metrics_logging() - mlflow.set_system_metrics_sampling_interval(None) - mlflow.set_system_metrics_samples_before_logging(None) + +def test_automatic_system_metrics_monitor_resume_existing_run(): + mlflow.enable_system_metrics_logging() + mlflow.set_system_metrics_sampling_interval(0.2) + mlflow.set_system_metrics_samples_before_logging(2) + with mlflow.start_run() as run: + time.sleep(2) + + # Pause for a bit to allow the system metrics monitoring to exit. + time.sleep(1) + thread_names = [thread.name for thread in threading.enumerate()] + # Check the system metrics monitoring thread has exited. + assert "SystemMetricsMonitor" not in thread_names + + # Get the last step. + metrics_history = mlflow.MlflowClient().get_metric_history( + run.info.run_id, "system/cpu_utilization_percentage" + ) + last_step = metrics_history[-1].step + + with mlflow.start_run(run.info.run_id) as run: + time.sleep(2) + mlflow_run = mlflow.get_run(run.info.run_id) + metrics = mlflow_run.data.metrics + + expected_metrics_name = [ + "cpu_utilization_percentage", + "system_memory_usage_megabytes", + "disk_usage_percentage", + "disk_usage_megabytes", + "disk_available_megabytes", + "network_receive_megabytes", + "network_transmit_megabytes", + ] + expected_metrics_name = [f"system/{name}" for name in expected_metrics_name] + for name in expected_metrics_name: + assert name in metrics + + # Check the step is correctly resumed. + metrics_history = mlflow.MlflowClient().get_metric_history( + run.info.run_id, "system/cpu_utilization_percentage" + ) + assert metrics_history[-1].step > last_step diff --git a/tests/tracking/test_rest_tracking.py b/tests/tracking/test_rest_tracking.py index 98930af66a5e8..3e05861aa305e 100644 --- a/tests/tracking/test_rest_tracking.py +++ b/tests/tracking/test_rest_tracking.py @@ -531,9 +531,6 @@ def test_validate_path_is_safe_good(path): "path", [ # relative path from current directory of C: drive - "C:path", - "C:path/", - "C:path/to/file", ".../...//", ], ) @@ -579,7 +576,11 @@ def test_validate_path_is_safe_bad(path): r".\..\path", r"path\..\to\file", r"path\..\..\to\file", - # Drive-relative absolute paths + # Drive-relative paths + r"C:path", + r"C:path/", + r"C:path/to/file", + r"C:../path/to/file", r"C:\path", r"C:/path", r"C:\path\to\file",