From bb31046e0ce361dac81342589ae636ba5d36399b Mon Sep 17 00:00:00 2001 From: Javier Martinez Date: Wed, 17 Jul 2024 09:05:37 +0200 Subject: [PATCH 1/5] docs: add troubleshooting --- fern/docs.yml | 2 + fern/docs/pages/installation/installation.mdx | 2 + .../pages/installation/troubleshooting.mdx | 44 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 fern/docs/pages/installation/troubleshooting.mdx diff --git a/fern/docs.yml b/fern/docs.yml index 5be021726..b8c8f06c4 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -41,6 +41,8 @@ navigation: path: ./docs/pages/installation/concepts.mdx - page: Installation path: ./docs/pages/installation/installation.mdx + - page: Troubleshooting + path: ./docs/pages/installation/troubleshooting.mdx # Manual of privateGPT: how to use it and configure it - tab: manual layout: diff --git a/fern/docs/pages/installation/installation.mdx b/fern/docs/pages/installation/installation.mdx index d1d186349..81213b6c0 100644 --- a/fern/docs/pages/installation/installation.mdx +++ b/fern/docs/pages/installation/installation.mdx @@ -81,6 +81,8 @@ set PGPT_PROFILES=ollama make run ``` +Refer to the [troubleshooting](./troubleshooting) section for specific issues you might encounter. + ### Local, Ollama-powered setup - RECOMMENDED **The easiest way to run PrivateGPT fully locally** is to depend on Ollama for the LLM. Ollama provides local LLM and Embeddings super easy to install and use, abstracting the complexity of GPU support. It's the recommended setup for local development. diff --git a/fern/docs/pages/installation/troubleshooting.mdx b/fern/docs/pages/installation/troubleshooting.mdx new file mode 100644 index 000000000..207bb0796 --- /dev/null +++ b/fern/docs/pages/installation/troubleshooting.mdx @@ -0,0 +1,44 @@ +# Downloading Gated and Private Models + +Many models are gated or private, requiring special access to use them. Follow these steps to gain access and set up your environment for using these models. + +## Accessing Gated Models + +1. **Request Access:** + Follow the instructions provided [here](https://huggingface.co/docs/hub/en/models-gated) to request access to the gated model. + +2. **Generate a Token:** + Once you have access, generate a token by following the instructions [here](https://huggingface.co/docs/hub/en/security-tokens). + +3. **Set the Token:** + Add the generated token to your `settings.yaml` file: + + ```yaml + huggingface: + access_token: + ``` + + Alternatively, set the `HF_TOKEN` environment variable: + + ```bash + export HF_TOKEN= + ``` + +# Tokenizer Setup + +PrivateGPT uses the `AutoTokenizer` library to tokenize input text accurately. It connects to HuggingFace's API to download the appropriate tokenizer for the specified model. + +## Configuring the Tokenizer + +1. **Specify the Model:** + In your `settings.yaml` file, specify the model you want to use: + + ```yaml + llm: + tokenizer: mistralai/Mistral-7B-Instruct-v0.2 + ``` + +2. **Set Access Token for Gated Models:** + If you are using a gated model, ensure the `access_token` is set as mentioned in the previous section. + +This configuration ensures that PrivateGPT can download and use the correct tokenizer for the model you are working with. \ No newline at end of file From 248266c275b62e49c079b2e7ed347d29ecae3a26 Mon Sep 17 00:00:00 2001 From: Javier Martinez Date: Wed, 17 Jul 2024 09:06:20 +0200 Subject: [PATCH 2/5] fix: pass HF token to setup script and prevent to download tokenizer when it is empty --- scripts/setup | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/setup b/scripts/setup index edba10498..ce8e833f0 100755 --- a/scripts/setup +++ b/scripts/setup @@ -24,6 +24,7 @@ snapshot_download( repo_id=settings().huggingface.embedding_hf_model_name, cache_dir=models_cache_path, local_dir=embedding_path, + token=settings().huggingface.access_token, ) print("Embedding model downloaded!") @@ -35,15 +36,18 @@ hf_hub_download( cache_dir=models_cache_path, local_dir=models_path, resume_download=resume_download, + token=settings().huggingface.access_token, ) print("LLM model downloaded!") # Download Tokenizer -print(f"Downloading tokenizer {settings().llm.tokenizer}") -AutoTokenizer.from_pretrained( - pretrained_model_name_or_path=settings().llm.tokenizer, - cache_dir=models_cache_path, -) -print("Tokenizer downloaded!") +if settings().llm.tokenizer: + print(f"Downloading tokenizer {settings().llm.tokenizer}") + AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=settings().llm.tokenizer, + cache_dir=models_cache_path, + token=settings().huggingface.access_token, + ) + print("Tokenizer downloaded!") print("Setup done") From 8d4c414b7ff3576a4ff8746268ae58ab6230ed37 Mon Sep 17 00:00:00 2001 From: Javier Martinez Date: Wed, 17 Jul 2024 09:07:18 +0200 Subject: [PATCH 3/5] fix: improve log and disable specific tokenizer by default --- private_gpt/components/llm/llm_component.py | 8 ++++---- settings.yaml | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/private_gpt/components/llm/llm_component.py b/private_gpt/components/llm/llm_component.py index bc66525e4..4388349fa 100644 --- a/private_gpt/components/llm/llm_component.py +++ b/private_gpt/components/llm/llm_component.py @@ -35,10 +35,10 @@ def __init__(self, settings: Settings) -> None: ) except Exception as e: logger.warning( - "Failed to download tokenizer %s. Falling back to " - "default tokenizer.", - settings.llm.tokenizer, - e, + f"Failed to download tokenizer {settings.llm.tokenizer}: {str(e)}" + f"Please follow the instructions in the documentation to download it if needed: " + f"https://docs.privategpt.dev/installation/getting-started/troubleshooting#tokenizer-setup." + f"Falling back to default tokenizer." ) logger.info("Initializing the LLM in mode=%s", llm_mode) diff --git a/settings.yaml b/settings.yaml index 8d882f73c..f28b9bd75 100644 --- a/settings.yaml +++ b/settings.yaml @@ -40,7 +40,8 @@ llm: # Should be matching the selected model max_new_tokens: 512 context_window: 3900 - tokenizer: mistralai/Mistral-7B-Instruct-v0.2 + # Select your tokenizer. Llama-index tokenizer is the default. + # tokenizer: mistralai/Mistral-7B-Instruct-v0.2 temperature: 0.1 # The temperature of the model. Increasing the temperature will make the model answer more creatively. A value of 0.1 would be more factual. (Default: 0.1) rag: From 68ba48d949bc10e0ea71d4499cb241a740bcdf7c Mon Sep 17 00:00:00 2001 From: Javier Martinez Date: Wed, 17 Jul 2024 09:10:10 +0200 Subject: [PATCH 4/5] chore: change HF_TOKEN environment to be aligned with default config --- settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.yaml b/settings.yaml index f28b9bd75..f29a4a837 100644 --- a/settings.yaml +++ b/settings.yaml @@ -77,7 +77,7 @@ embedding: huggingface: embedding_hf_model_name: BAAI/bge-small-en-v1.5 - access_token: ${HUGGINGFACE_TOKEN:} + access_token: ${HF_TOKEN:} vectorstore: database: qdrant From f716e729ba0d7e549774605c902c10ac64a4dc72 Mon Sep 17 00:00:00 2001 From: Javier Martinez Date: Wed, 17 Jul 2024 09:59:30 +0200 Subject: [PATCH 5/5] ifx: mypy --- private_gpt/components/llm/llm_component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private_gpt/components/llm/llm_component.py b/private_gpt/components/llm/llm_component.py index 4388349fa..d4ab81f43 100644 --- a/private_gpt/components/llm/llm_component.py +++ b/private_gpt/components/llm/llm_component.py @@ -35,7 +35,7 @@ def __init__(self, settings: Settings) -> None: ) except Exception as e: logger.warning( - f"Failed to download tokenizer {settings.llm.tokenizer}: {str(e)}" + f"Failed to download tokenizer {settings.llm.tokenizer}: {e!s}" f"Please follow the instructions in the documentation to download it if needed: " f"https://docs.privategpt.dev/installation/getting-started/troubleshooting#tokenizer-setup." f"Falling back to default tokenizer."