Merge pull request #96 from biocypher/benchmark_results_docs_page

issue #93: show benchmark results on docs page (first draft)

.gitignore

@@ -11,4 +11,5 @@ __pycache__/
 .idea/
 *.env
 volumes/
-benchmark/encrypted_llm_test_data.json
+benchmark/encrypted_llm_test_data.json
+site/

DEVELOPER.md

@@ -46,7 +46,7 @@ For ensuring code quality, the following tools are used:
 
 - [black](https://black.readthedocs.io/en/stable/) for automated code formatting
 
-<!-- - [pre-commit-hooks](https://github.com/pre-commit/pre-commit-hooks) for
+- [pre-commit-hooks](https://github.com/pre-commit/pre-commit-hooks) for
 ensuring some general rules
 
 - [pep585-upgrade](https://github.com/snok/pep585-upgrade) for automatically
@@ -55,28 +55,26 @@ upgrading type hints to the new native types defined in PEP 585
 - [pygrep-hooks](https://github.com/pre-commit/pygrep-hooks) for ensuring some
 general naming rules -->
 
-<!-- Pre-commit hooks are used to automatically run these tools before each commit.
+Pre-commit hooks are used to automatically run these tools before each commit.
 They are defined in [.pre-commit-config.yaml](./.pre-commit-config.yaml). To
 install the hooks run `poetry run pre-commit install`. The hooks are then
 executed before each commit. For running the hook for all project files (not
 only the changed ones) run `poetry run pre-commit run --all-files`. -->
 
-<!-- The project uses a [Sphinx](https://www.sphinx-doc.org/en/master/) autodoc
-GitHub Actions workflow to generate the documentation. If you add new code,
+The project uses [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) within a GitHub Actions workflow to generate the documentation. If you add new code,
 please make sure that it is documented accordingly and in a consistent manner
 with the existing code base. The docstrings should follow the [Google style
 guide](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html).
 To check if the docs build successfully, you can build them locally by running
-`make html` in the `docs` directory. -->
-
+`mkdocs build` in the project root directory. To preview your changes run `mkdocs serve`.
+`
+<!-- TODO: doctest -->
 <!-- When adding new code snippets to the documentation, make sure that they are
 automatically tested with
 [doctest](https://sphinx-tutorial.readthedocs.io/step-3/#testing-your-code);
 this ensures that no outdated code snippets are part of the documentation. -->
 
-Documentation currently lives in the repository's
-[wiki](https://github.com/biocypher/biochatter/wiki). We will soon create a
-Sphinx-based documentation site.
+The documentation is hosted [here](https://biochatter.org/).
 
 
 ## Testing

benchmark/benchmark_utils.py

@@ -1,6 +1,7 @@
-import pandas as pd
 import pytest
 
+import pandas as pd
+
 
 def benchmark_already_executed(
     model_name: str,
@@ -63,24 +64,27 @@ def return_or_create_result_file(
     try:
         results = pd.read_csv(file_path, header=0)
     except (pd.errors.EmptyDataError, FileNotFoundError):
-        results = pd.DataFrame(columns=["model_name", "subtask", "score"])
+        results = pd.DataFrame(
+            columns=["model_name", "subtask", "score", "iterations"]
+        )
         results.to_csv(file_path, index=False)
     return results
 
 
 def write_results_to_file(
-    model_name: str, subtask: str, score: str, file_path: str
+    model_name: str, subtask: str, score: str, iterations: str, file_path: str
 ):
     """Writes the benchmark results for the subtask to the result file.
 
     Args:
         model_name (str): The model name, e.g. "gpt-3.5-turbo"
         subtask (str): The benchmark subtask test case, e.g. "entities_0"
         score (str): The benchmark score, e.g. "1/1"
+        iterations (str): The number of iterations, e.g. "1"
     """
     results = pd.read_csv(file_path, header=0)
     new_row = pd.DataFrame(
-        [[model_name, subtask, score]], columns=results.columns
+        [[model_name, subtask, score, iterations]], columns=results.columns
     )
     results = pd.concat([results, new_row], ignore_index=True).sort_values(
         by=["model_name", "subtask"]

benchmark/conftest.py

@@ -1,5 +1,6 @@
 import os
 
+from xinference.client import Client
 import pytest
 
 import numpy as np
@@ -8,20 +9,14 @@
 from biochatter.prompts import BioCypherPromptEngine
 from benchmark.load_dataset import get_benchmark_dataset
 from biochatter.llm_connect import GptConversation, XinferenceConversation
-
-from xinference.client import Client
-
-from .benchmark_utils import (
-    benchmark_already_executed,
-)
+from .benchmark_utils import benchmark_already_executed
 
 # how often should each benchmark be run?
 N_ITERATIONS = 2
 
 # which dataset should be used for benchmarking?
 BENCHMARK_DATASET = get_benchmark_dataset()
 
-
 # which models should be benchmarked?
 OPENAI_MODEL_NAMES = [
     "gpt-3.5-turbo",
@@ -259,14 +254,20 @@ def result_files():
             result_file = pd.read_csv(file, header=0)
         except (pd.errors.EmptyDataError, FileNotFoundError):
             result_file = pd.DataFrame(
-                columns=["model_name", "subtask", "score"]
+                columns=["model_name", "subtask", "score", "iterations"]
             )
             result_file.to_csv(file, index=False)
 
         if not np.array_equal(
-            result_file.columns, ["model_name", "subtask", "score"]
+            result_file.columns,
+            ["model_name", "subtask", "score", "iterations"],
         ):
-            result_file.columns = ["model_name", "subtask", "score"]
+            result_file.columns = [
+                "model_name",
+                "subtask",
+                "score",
+                "iterations",
+            ]
 
         result_files[file] = result_file
 

benchmark/results/end_to_end_query_generation.csv

@@ -1,29 +1,29 @@
-model_name,subtask,score
-gpt-3.5-turbo,0_single_word,7.0/7;2
-gpt-3.5-turbo,1_multi_word,7.0/7;2
-gpt-4,0_single_word,7.0/7;2
-gpt-4,1_multi_word,7.0/7;2
-llama-2-chat:13:ggmlv3:q2_K,0_single_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q2_K,1_multi_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q4_0,0_single_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q4_0,1_multi_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q5_0,0_single_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q5_0,1_multi_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q8_0,0_single_word,0.0/7;2
-llama-2-chat:13:ggmlv3:q8_0,1_multi_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q2_K,0_single_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q2_K,1_multi_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q4_0,0_single_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q4_0,1_multi_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q5_0,0_single_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q5_0,1_multi_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q8_0,0_single_word,0.0/7;2
-llama-2-chat:7:ggmlv3:q8_0,1_multi_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,0_single_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,1_multi_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,0_single_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,1_multi_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,0_single_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,1_multi_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,0_single_word,0.0/7;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,1_multi_word,0.0/7;2
+model_name,subtask,score,iterations
+gpt-3.5-turbo,0_single_word,7.0/7,2
+gpt-3.5-turbo,1_multi_word,7.0/7,2
+gpt-4,0_single_word,7.0/7,2
+gpt-4,1_multi_word,7.0/7,2
+llama-2-chat:13:ggmlv3:q2_K,0_single_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q2_K,1_multi_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q4_0,0_single_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q4_0,1_multi_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q5_0,0_single_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q5_0,1_multi_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q8_0,0_single_word,0.0/7,2
+llama-2-chat:13:ggmlv3:q8_0,1_multi_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q2_K,0_single_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q2_K,1_multi_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q4_0,0_single_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q4_0,1_multi_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q5_0,0_single_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q5_0,1_multi_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q8_0,0_single_word,0.0/7,2
+llama-2-chat:7:ggmlv3:q8_0,1_multi_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,0_single_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,1_multi_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,0_single_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,1_multi_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,0_single_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,1_multi_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,0_single_word,0.0/7,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,1_multi_word,0.0/7,2

benchmark/results/entity_selection.csv

@@ -1,29 +1,29 @@
-model_name,subtask,score
-gpt-3.5-turbo,0_single_word,2.0/2;2
-gpt-3.5-turbo,1_multi_word,2.0/2;2
-gpt-4,0_single_word,2.0/2;2
-gpt-4,1_multi_word,2.0/2;2
-llama-2-chat:13:ggmlv3:q2_K,0_single_word,0.0/2;2
-llama-2-chat:13:ggmlv3:q2_K,1_multi_word,0.0/2;2
-llama-2-chat:13:ggmlv3:q4_0,0_single_word,1.0/2;2
-llama-2-chat:13:ggmlv3:q4_0,1_multi_word,0.0/2;2
-llama-2-chat:13:ggmlv3:q5_0,0_single_word,0.0/2;2
-llama-2-chat:13:ggmlv3:q5_0,1_multi_word,0.0/2;2
-llama-2-chat:13:ggmlv3:q8_0,0_single_word,0.0/2;2
-llama-2-chat:13:ggmlv3:q8_0,1_multi_word,0.0/2;2
-llama-2-chat:7:ggmlv3:q2_K,0_single_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q2_K,1_multi_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q4_0,0_single_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q4_0,1_multi_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q5_0,0_single_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q5_0,1_multi_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q8_0,0_single_word,1.0/2;2
-llama-2-chat:7:ggmlv3:q8_0,1_multi_word,1.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,0_single_word,0.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,1_multi_word,0.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,0_single_word,0.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,1_multi_word,0.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,0_single_word,0.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,1_multi_word,1.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,0_single_word,0.0/2;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,1_multi_word,1.0/2;2
+model_name,subtask,score,iterations
+gpt-3.5-turbo,0_single_word,2.0/2,2
+gpt-3.5-turbo,1_multi_word,2.0/2,2
+gpt-4,0_single_word,2.0/2,2
+gpt-4,1_multi_word,2.0/2,2
+llama-2-chat:13:ggmlv3:q2_K,0_single_word,0.0/2,2
+llama-2-chat:13:ggmlv3:q2_K,1_multi_word,0.0/2,2
+llama-2-chat:13:ggmlv3:q4_0,0_single_word,1.0/2,2
+llama-2-chat:13:ggmlv3:q4_0,1_multi_word,0.0/2,2
+llama-2-chat:13:ggmlv3:q5_0,0_single_word,0.0/2,2
+llama-2-chat:13:ggmlv3:q5_0,1_multi_word,0.0/2,2
+llama-2-chat:13:ggmlv3:q8_0,0_single_word,0.0/2,2
+llama-2-chat:13:ggmlv3:q8_0,1_multi_word,0.0/2,2
+llama-2-chat:7:ggmlv3:q2_K,0_single_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q2_K,1_multi_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q4_0,0_single_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q4_0,1_multi_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q5_0,0_single_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q5_0,1_multi_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q8_0,0_single_word,1.0/2,2
+llama-2-chat:7:ggmlv3:q8_0,1_multi_word,1.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,0_single_word,0.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,1_multi_word,0.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,0_single_word,0.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,1_multi_word,0.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,0_single_word,0.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,1_multi_word,1.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,0_single_word,0.0/2,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,1_multi_word,1.0/2,2

benchmark/results/explicit_relevance_of_single_fragments.csv

@@ -1,57 +1,57 @@
-model_name,subtask,score
-gpt-3.5-turbo,2_explicit,1.0/1;2
-gpt-3.5-turbo,3_explicit,0.0/1;2
-gpt-3.5-turbo,4_explicit,1.0/1;2
-gpt-3.5-turbo,5_explicit,1.0/1;2
-gpt-4,2_explicit,0.0/1;2
-gpt-4,3_explicit,0.0/1;2
-gpt-4,4_explicit,0.0/1;2
-gpt-4,5_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q2_K,2_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q2_K,3_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q2_K,4_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q2_K,5_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q4_0,2_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q4_0,3_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q4_0,4_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q4_0,5_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q5_0,2_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q5_0,3_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q5_0,4_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q5_0,5_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q8_0,2_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q8_0,3_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q8_0,4_explicit,0.0/1;2
-llama-2-chat:13:ggmlv3:q8_0,5_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q2_K,2_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q2_K,3_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q2_K,4_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q2_K,5_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q4_0,2_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q4_0,3_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q4_0,4_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q4_0,5_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q5_0,2_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q5_0,3_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q5_0,4_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q5_0,5_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q8_0,2_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q8_0,3_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q8_0,4_explicit,0.0/1;2
-llama-2-chat:7:ggmlv3:q8_0,5_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,2_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,3_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,4_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,5_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,2_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,3_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,4_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,5_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,2_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,3_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,4_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,5_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,2_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,3_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,4_explicit,0.0/1;2
-mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,5_explicit,0.0/1;2
+model_name,subtask,score,iterations
+gpt-3.5-turbo,2_explicit,1.0/1,2
+gpt-3.5-turbo,3_explicit,0.0/1,2
+gpt-3.5-turbo,4_explicit,1.0/1,2
+gpt-3.5-turbo,5_explicit,1.0/1,2
+gpt-4,2_explicit,0.0/1,2
+gpt-4,3_explicit,0.0/1,2
+gpt-4,4_explicit,0.0/1,2
+gpt-4,5_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q2_K,2_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q2_K,3_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q2_K,4_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q2_K,5_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q4_0,2_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q4_0,3_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q4_0,4_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q4_0,5_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q5_0,2_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q5_0,3_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q5_0,4_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q5_0,5_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q8_0,2_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q8_0,3_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q8_0,4_explicit,0.0/1,2
+llama-2-chat:13:ggmlv3:q8_0,5_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q2_K,2_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q2_K,3_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q2_K,4_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q2_K,5_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q4_0,2_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q4_0,3_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q4_0,4_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q4_0,5_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q5_0,2_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q5_0,3_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q5_0,4_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q5_0,5_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q8_0,2_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q8_0,3_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q8_0,4_explicit,0.0/1,2
+llama-2-chat:7:ggmlv3:q8_0,5_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,2_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,3_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,4_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q2_K,5_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,2_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,3_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,4_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q4_0,5_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,2_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,3_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,4_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q5_0,5_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,2_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,3_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,4_explicit,0.0/1,2
+mixtral-instruct-v0.1:46_7:ggufv2:Q8_0,5_explicit,0.0/1,2