From 15bd2bbcdfa2a3508b0a1fb070dadafaa0623ad7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 12 Jan 2024 09:23:41 +0000 Subject: [PATCH 01/10] local_llm_fixes --- Dockerfile | 3 +- Dockerfile-gpu | 45 +++++++++++++++++ docker-compose-gpu.yml | 95 +++++++++++++++++++++++++++++++++++ docker-compose.yaml | 9 +++- requirements.txt | 1 - superagi/helper/llm_loader.py | 2 +- 6 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 Dockerfile-gpu create mode 100644 docker-compose-gpu.yml diff --git a/Dockerfile b/Dockerfile index 0e9a2fa69..e1a24d011 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ ENV PATH="/opt/venv/bin:$PATH" COPY requirements.txt . RUN pip install --upgrade pip && \ pip install --no-cache-dir -r requirements.txt +RUN python3 -m pip install llama-cpp-python RUN python3.10 -c "import nltk; nltk.download('punkt')" && \ python3.10 -c "import nltk; nltk.download('averaged_perceptron_tagger')" @@ -36,4 +37,4 @@ COPY --from=compile-image /root/nltk_data /root/nltk_data ENV PATH="/opt/venv/bin:$PATH" -EXPOSE 8001 +EXPOSE 8001 \ No newline at end of file diff --git a/Dockerfile-gpu b/Dockerfile-gpu new file mode 100644 index 000000000..3cb7e0858 --- /dev/null +++ b/Dockerfile-gpu @@ -0,0 +1,45 @@ +# Define the CUDA SDK version you need +ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04" +FROM nvidia/cuda:${CUDA_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /app + +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y git build-essential \ + python3 python3-pip python3.10-venv libpq-dev gcc wget \ + ocl-icd-opencl-dev opencl-headers clinfo \ + libclblast-dev libopenblas-dev \ + && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +# Create a virtual environment and activate it +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Install Python dependencies from requirements.txt +COPY requirements.txt . +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# Running nltk setup as you mentioned +RUN python3.10 -c "import nltk; nltk.download('punkt')" && \ + python3.10 -c "import nltk; nltk.download('averaged_perceptron_tagger')" + +# Copy the application code +COPY . . + +ENV CUDA_DOCKER_ARCH=all +ENV LLAMA_CUBLAS=1 + +RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python + +# Make necessary scripts executable +RUN chmod +x ./entrypoint.sh ./wait-for-it.sh ./install_tool_dependencies.sh ./entrypoint_celery.sh + +# Set environment variable to point to the custom libllama.so +# ENV LLAMA_CPP_LIB=/app/llama.cpp/libllama.so + +EXPOSE 8001 + +CMD ["./entrypoint.sh"] \ No newline at end of file diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml new file mode 100644 index 000000000..4b439eb9f --- /dev/null +++ b/docker-compose-gpu.yml @@ -0,0 +1,95 @@ +version: '3.8' +services: + backend: + volumes: + - "./:/app" + build: + context: . + dockerfile: Dockerfile-gpu # Correct nested structure under build key + depends_on: + - super__redis + - super__postgres + networks: + - super_network + command: ["/app/wait-for-it.sh", "super__postgres:5432","-t","60","--","/app/entrypoint.sh"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + celery: + volumes: + - "./:/app" + - "${EXTERNAL_RESOURCE_DIR:-./workspace}:/app/ext" + build: + context: . + dockerfile: Dockerfile-gpu # Correct nested structure under build key + depends_on: + - super__redis + - super__postgres + networks: + - super_network + command: ["/app/entrypoint_celery.sh"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + gui: + build: + context: ./gui + args: + NEXT_PUBLIC_API_BASE_URL: "/api" + networks: + - super_network +# volumes: +# - ./gui:/app +# - /app/node_modules/ +# - /app/.next/ + super__redis: + image: "redis/redis-stack-server:latest" + networks: + - super_network +# uncomment to expose redis port to host +# ports: +# - "6379:6379" + volumes: + - redis_data:/data + + super__postgres: + image: "docker.io/library/postgres:15" + environment: + - POSTGRES_USER=superagi + - POSTGRES_PASSWORD=password + - POSTGRES_DB=super_agi_main + volumes: + - superagi_postgres_data:/var/lib/postgresql/data/ + networks: + - super_network +# uncomment to expose postgres port to host +# ports: +# - "5432:5432" + + proxy: + image: nginx:stable-alpine + ports: + - "3000:80" + networks: + - super_network + depends_on: + - backend + - gui + volumes: + - ./nginx/default.conf:/etc/nginx/conf.d/default.conf + +networks: + super_network: + driver: bridge +volumes: + superagi_postgres_data: + redis_data: \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 926b8515c..8314047e5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,18 +3,23 @@ services: backend: volumes: - "./:/app" - build: . + build: + context: . + dockerfile: Dockerfile # Correct nested structure under build key depends_on: - super__redis - super__postgres networks: - super_network command: ["/app/wait-for-it.sh", "super__postgres:5432","-t","60","--","/app/entrypoint.sh"] + celery: volumes: - "./:/app" - "${EXTERNAL_RESOURCE_DIR:-./workspace}:/app/ext" - build: . + build: + context: . + dockerfile: Dockerfile # Correct nested structure under build key depends_on: - super__redis - super__postgres diff --git a/requirements.txt b/requirements.txt index 1a2892c25..ab45bb1c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -158,4 +158,3 @@ google-generativeai==0.1.0 unstructured==0.8.1 ai21==1.2.6 typing-extensions==4.5.0 -llama_cpp_python==0.2.7 diff --git a/superagi/helper/llm_loader.py b/superagi/helper/llm_loader.py index 8d78337da..d5d1b98d3 100644 --- a/superagi/helper/llm_loader.py +++ b/superagi/helper/llm_loader.py @@ -22,7 +22,7 @@ def model(self): if self._model is None: try: self._model = Llama( - model_path="/app/local_model_path", n_ctx=self.context_length) + model_path="/app/local_model_path", n_ctx=self.context_length, n_gpu_layers=get_config('GPU_LAYERS')) except Exception as e: logger.error(e) return self._model From 21788547dba379664dc07f973fdbfd39d48605cd Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 09:27:00 +0000 Subject: [PATCH 02/10] local_llm_fixes --- docker-compose-gpu.yml | 4 ++-- docker-compose.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml index 4b439eb9f..87e8ead0d 100644 --- a/docker-compose-gpu.yml +++ b/docker-compose-gpu.yml @@ -5,7 +5,7 @@ services: - "./:/app" build: context: . - dockerfile: Dockerfile-gpu # Correct nested structure under build key + dockerfile: Dockerfile-gpu depends_on: - super__redis - super__postgres @@ -26,7 +26,7 @@ services: - "${EXTERNAL_RESOURCE_DIR:-./workspace}:/app/ext" build: context: . - dockerfile: Dockerfile-gpu # Correct nested structure under build key + dockerfile: Dockerfile-gpu depends_on: - super__redis - super__postgres diff --git a/docker-compose.yaml b/docker-compose.yaml index 8314047e5..8b8725735 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,7 +5,7 @@ services: - "./:/app" build: context: . - dockerfile: Dockerfile # Correct nested structure under build key + dockerfile: Dockerfile depends_on: - super__redis - super__postgres @@ -19,7 +19,7 @@ services: - "${EXTERNAL_RESOURCE_DIR:-./workspace}:/app/ext" build: context: . - dockerfile: Dockerfile # Correct nested structure under build key + dockerfile: Dockerfile depends_on: - super__redis - super__postgres From 4daf18acf54ea9a403fc8496978fc4b075424872 Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 09:28:53 +0000 Subject: [PATCH 03/10] local_llm_fixes --- config_template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config_template.yaml b/config_template.yaml index b3a0906e9..04c35e3f2 100644 --- a/config_template.yaml +++ b/config_template.yaml @@ -122,3 +122,4 @@ ENGINE_ID: "stable-diffusion-xl-beta-v2-2-2" ## To use Qdrant for vector store #QDRANT_HOST_NAME: YOUR_QDRANT_HOST_NAME #QDRANT_PORT: YOUR_QDRANT_PORT +GPU_LAYERS: GPU LAYERS THAT YOU WANT TO OFFLOAD TO THE GPU WHILE USING LOCAL LLMS From b7f4540b5e63cf8088c797c34c398e52feee0857 Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 10:25:51 +0000 Subject: [PATCH 04/10] local_llms_fix --- Dockerfile | 2 +- Dockerfile-gpu | 2 +- docker-compose-gpu.yml | 2 +- requirements.txt | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1a24d011..c0645cf9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PATH="/opt/venv/bin:$PATH" COPY requirements.txt . RUN pip install --upgrade pip && \ pip install --no-cache-dir -r requirements.txt -RUN python3 -m pip install llama-cpp-python +RUN python3 -m pip install llama-cpp-python==0.2.7 --force-reinstall --upgrade --no-cache-dir RUN python3.10 -c "import nltk; nltk.download('punkt')" && \ python3.10 -c "import nltk; nltk.download('averaged_perceptron_tagger')" diff --git a/Dockerfile-gpu b/Dockerfile-gpu index 3cb7e0858..0b11e1b41 100644 --- a/Dockerfile-gpu +++ b/Dockerfile-gpu @@ -32,7 +32,7 @@ COPY . . ENV CUDA_DOCKER_ARCH=all ENV LLAMA_CUBLAS=1 -RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python==0.2.7 --force-reinstall --upgrade --no-cache-dir # Make necessary scripts executable RUN chmod +x ./entrypoint.sh ./wait-for-it.sh ./install_tool_dependencies.sh ./entrypoint_celery.sh diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml index 87e8ead0d..2428b0b8a 100644 --- a/docker-compose-gpu.yml +++ b/docker-compose-gpu.yml @@ -38,7 +38,7 @@ services: reservations: devices: - driver: nvidia - count: 1 + count: all capabilities: [gpu] gui: build: diff --git a/requirements.txt b/requirements.txt index ab45bb1c7..9ebdd1a49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -158,3 +158,4 @@ google-generativeai==0.1.0 unstructured==0.8.1 ai21==1.2.6 typing-extensions==4.5.0 +llama_cpp_python==0.2.7 \ No newline at end of file From 706b96a94465c14fa5feebc7b42372982332943d Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 10:28:30 +0000 Subject: [PATCH 05/10] local_llms_fix --- config_template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_template.yaml b/config_template.yaml index 04c35e3f2..40ad6a8ed 100644 --- a/config_template.yaml +++ b/config_template.yaml @@ -122,4 +122,4 @@ ENGINE_ID: "stable-diffusion-xl-beta-v2-2-2" ## To use Qdrant for vector store #QDRANT_HOST_NAME: YOUR_QDRANT_HOST_NAME #QDRANT_PORT: YOUR_QDRANT_PORT -GPU_LAYERS: GPU LAYERS THAT YOU WANT TO OFFLOAD TO THE GPU WHILE USING LOCAL LLMS +#GPU_LAYERS: GPU LAYERS THAT YOU WANT TO OFFLOAD TO THE GPU WHILE USING LOCAL LLMS From 360ef25026760d654e8dd065456ff458644767d6 Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 10:50:34 +0000 Subject: [PATCH 06/10] fixes in duckduckgo test --- .../tools/duck_duck_go/test_duckduckgo_results.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/tools/duck_duck_go/test_duckduckgo_results.py b/tests/unit_tests/tools/duck_duck_go/test_duckduckgo_results.py index 61a551fc6..12684324f 100644 --- a/tests/unit_tests/tools/duck_duck_go/test_duckduckgo_results.py +++ b/tests/unit_tests/tools/duck_duck_go/test_duckduckgo_results.py @@ -1,3 +1,5 @@ +import unittest +from unittest.mock import patch import pytest from superagi.tools.duck_duck_go.duck_duck_go_search import DuckDuckGoSearchTool @@ -11,9 +13,13 @@ def test_get_raw_duckduckgo_results_empty_query(self): result = self.your_obj.get_raw_duckduckgo_results(query) assert result == expected_result - def test_get_raw_duckduckgo_results_valid_query(self): + @patch('superagi.tools.duck_duck_go.duck_duck_go_search.DuckDuckGoSearchTool.get_raw_duckduckgo_results') + def test_get_raw_duckduckgo_results_valid_query(self, mock_get_raw_duckduckgo_results): query = "python" expected_result_length = 10 + mock_results = ['result1', 'result2', 'result3', 'result4', 'result5', + 'result6', 'result7', 'result8', 'result9', 'result10'] + mock_get_raw_duckduckgo_results.return_value = mock_results result = self.your_obj.get_raw_duckduckgo_results(query) assert len(result) == expected_result_length From b141ce75a45f704930dadc2ced68f1c6e1d4a46e Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 13:30:21 +0000 Subject: [PATCH 07/10] local_llm_fixes --- docker-compose.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 8b8725735..926b8515c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,23 +3,18 @@ services: backend: volumes: - "./:/app" - build: - context: . - dockerfile: Dockerfile + build: . depends_on: - super__redis - super__postgres networks: - super_network command: ["/app/wait-for-it.sh", "super__postgres:5432","-t","60","--","/app/entrypoint.sh"] - celery: volumes: - "./:/app" - "${EXTERNAL_RESOURCE_DIR:-./workspace}:/app/ext" - build: - context: . - dockerfile: Dockerfile + build: . depends_on: - super__redis - super__postgres From 2a7559375570c3696a84ff70158b2fa29d4fa2b3 Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 13:55:26 +0000 Subject: [PATCH 08/10] local_llm_fixes --- Dockerfile | 1 - docker-compose-gpu.yml | 2 ++ superagi/helper/llm_loader.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c0645cf9a..3a5498cec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ ENV PATH="/opt/venv/bin:$PATH" COPY requirements.txt . RUN pip install --upgrade pip && \ pip install --no-cache-dir -r requirements.txt -RUN python3 -m pip install llama-cpp-python==0.2.7 --force-reinstall --upgrade --no-cache-dir RUN python3.10 -c "import nltk; nltk.download('punkt')" && \ python3.10 -c "import nltk; nltk.download('averaged_perceptron_tagger')" diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml index 2428b0b8a..02b1b447f 100644 --- a/docker-compose-gpu.yml +++ b/docker-compose-gpu.yml @@ -3,6 +3,7 @@ services: backend: volumes: - "./:/app" + - "/home/ubuntu/models/vicuna-7B-v1.5-GGUF/vicuna-7b-v1.5.Q5_K_M.gguf:/app/local_model_path" build: context: . dockerfile: Dockerfile-gpu @@ -24,6 +25,7 @@ services: volumes: - "./:/app" - "${EXTERNAL_RESOURCE_DIR:-./workspace}:/app/ext" + - "/home/ubuntu/models/vicuna-7B-v1.5-GGUF/vicuna-7b-v1.5.Q5_K_M.gguf:/app/local_model_path" build: context: . dockerfile: Dockerfile-gpu diff --git a/superagi/helper/llm_loader.py b/superagi/helper/llm_loader.py index d5d1b98d3..631065d93 100644 --- a/superagi/helper/llm_loader.py +++ b/superagi/helper/llm_loader.py @@ -22,7 +22,7 @@ def model(self): if self._model is None: try: self._model = Llama( - model_path="/app/local_model_path", n_ctx=self.context_length, n_gpu_layers=get_config('GPU_LAYERS')) + model_path="/app/local_model_path", n_ctx=self.context_length, n_gpu_layers=get_config('GPU_LAYERS', '-1')) except Exception as e: logger.error(e) return self._model From e8a306def5ce9d6d595299c8345340123e2037b7 Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 14:00:10 +0000 Subject: [PATCH 09/10] local_llm_fixes --- README.MD | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.MD b/README.MD index 23e0731c0..a220960b8 100644 --- a/README.MD +++ b/README.MD @@ -126,10 +126,15 @@ cd SuperAGI 4. Ensure that Docker is installed on your system. You can download and install it from [here](https://docs.docker.com/get-docker/). -5. Once you have Docker Desktop running, run the following command in the in the SuperAGI directory : -``` -docker-compose up --build -``` +5. Once you have Docker Desktop running, run the following command in the SuperAGI directory: + a. For regular usage: + ``` + docker compose -f docker-compose.yaml up --build + ``` + b. If you want to use SuperAGI with Local LLMs and have GPU, run the following command: + ``` + docker compose -f docker-compose-gpu.yml up --build + ``` 6. Open your web browser and navigate to http://localhost:3000 to access SuperAGI. From c4e856ac103ecef00b34ef2e36ba83c73f2462a2 Mon Sep 17 00:00:00 2001 From: rounak bhatia Date: Fri, 12 Jan 2024 14:03:02 +0000 Subject: [PATCH 10/10] local_llm_fixes --- README.MD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.MD b/README.MD index a220960b8..74a0fe055 100644 --- a/README.MD +++ b/README.MD @@ -127,15 +127,18 @@ cd SuperAGI 4. Ensure that Docker is installed on your system. You can download and install it from [here](https://docs.docker.com/get-docker/). 5. Once you have Docker Desktop running, run the following command in the SuperAGI directory: + a. For regular usage: ``` docker compose -f docker-compose.yaml up --build ``` + b. If you want to use SuperAGI with Local LLMs and have GPU, run the following command: ``` docker compose -f docker-compose-gpu.yml up --build ``` + 6. Open your web browser and navigate to http://localhost:3000 to access SuperAGI. #### 🌀 Digital Ocean