diff --git a/.gitignore b/.gitignore index 06f43f8a..b4ac753d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,11 @@ application.local.yml llm_config.local.yml +###################### +# Docker +###################### +/docker/.docker-data/artemis-data/* +!/docker/.docker-data/artemis-data/.gitkeep ######################## # Auto-generated rules # diff --git a/app/config.py b/app/config.py index ae63c3a5..5984e7fb 100644 --- a/app/config.py +++ b/app/config.py @@ -8,9 +8,16 @@ class APIKeyConfig(BaseModel): token: str +class WeaviateSettings(BaseModel): + host: str + port: int + grpc_port: int + + class Settings(BaseModel): api_keys: list[APIKeyConfig] env_vars: dict[str, str] + weaviate: WeaviateSettings @classmethod def get_settings(cls): diff --git a/app/domain/data/image_message_content_dto.py b/app/domain/data/image_message_content_dto.py index a73e2654..532322dd 100644 --- a/app/domain/data/image_message_content_dto.py +++ b/app/domain/data/image_message_content_dto.py @@ -1,7 +1,8 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field, ConfigDict from typing import Optional class ImageMessageContentDTO(BaseModel): - base64: str - prompt: Optional[str] + base64: str = Field(..., alias="pdfFile") + prompt: Optional[str] = None + model_config = ConfigDict(populate_by_name=True) diff --git a/app/domain/data/lecture_unit_dto.py b/app/domain/data/lecture_unit_dto.py index 8b123c1c..26ef1785 100644 --- a/app/domain/data/lecture_unit_dto.py +++ b/app/domain/data/lecture_unit_dto.py @@ -3,11 +3,12 @@ class LectureUnitDTO(BaseModel): to_update: bool = Field(alias="toUpdate") - pdf_file_base64: str = Field(alias="pdfFile") + base_url: str = Field(alias="artemisBaseUrl") + pdf_file_base64: str = Field(default="", alias="pdfFile") lecture_unit_id: int = Field(alias="lectureUnitId") - lecture_unit_name: str = Field(alias="lectureUnitName") + lecture_unit_name: str = Field(default="", alias="lectureUnitName") lecture_id: int = Field(alias="lectureId") - lecture_name: str = Field(alias="lectureName") + lecture_name: str = Field(default="", alias="lectureName") course_id: int = Field(alias="courseId") - course_name: str = Field(alias="courseName") - course_description: str = Field(alias="courseDescription") + course_name: str = Field(default="", alias="courseName") + course_description: str = Field(default="", alias="courseDescription") diff --git a/app/domain/ingestion/__init__.py b/app/domain/ingestion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/domain/ingestion/ingestion_pipeline_execution_dto.py b/app/domain/ingestion/ingestion_pipeline_execution_dto.py new file mode 100644 index 00000000..e8a9882f --- /dev/null +++ b/app/domain/ingestion/ingestion_pipeline_execution_dto.py @@ -0,0 +1,12 @@ +from typing import List + +from pydantic import Field + +from app.domain import PipelineExecutionDTO +from app.domain.data.lecture_unit_dto import LectureUnitDTO + + +class IngestionPipelineExecutionDto(PipelineExecutionDTO): + lecture_units: List[LectureUnitDTO] = Field( + ..., alias="pyrisLectureUnitWebhookDTOS" + ) diff --git a/app/domain/ingestion/ingestion_status_update_dto.py b/app/domain/ingestion/ingestion_status_update_dto.py new file mode 100644 index 00000000..351b9e6f --- /dev/null +++ b/app/domain/ingestion/ingestion_status_update_dto.py @@ -0,0 +1,7 @@ +from typing import Optional + +from ...domain.status.status_update_dto import StatusUpdateDTO + + +class IngestionStatusUpdateDTO(StatusUpdateDTO): + result: Optional[str] = None diff --git a/app/domain/pipeline_execution_settings_dto.py b/app/domain/pipeline_execution_settings_dto.py index 5fc014ed..340e8783 100644 --- a/app/domain/pipeline_execution_settings_dto.py +++ b/app/domain/pipeline_execution_settings_dto.py @@ -5,5 +5,7 @@ class PipelineExecutionSettingsDTO(BaseModel): authentication_token: str = Field(alias="authenticationToken") - allowed_model_identifiers: List[str] = Field(alias="allowedModelIdentifiers") + allowed_model_identifiers: List[str] = Field( + default=[], alias="allowedModelIdentifiers" + ) artemis_base_url: str = Field(alias="artemisBaseUrl") diff --git a/app/ingestion/abstract_ingestion.py b/app/ingestion/abstract_ingestion.py index d78244f0..85bfba23 100644 --- a/app/ingestion/abstract_ingestion.py +++ b/app/ingestion/abstract_ingestion.py @@ -13,17 +13,3 @@ def chunk_data(self, path: str) -> List[Dict[str, str]]: Abstract method to chunk code files in the root directory. """ pass - - @abstractmethod - def ingest(self, path: str) -> bool: - """ - Abstract method to ingest repositories into the database. - """ - pass - - @abstractmethod - def update(self, path: str): - """ - Abstract method to update a repository in the database. - """ - pass diff --git a/app/llm/external/openai_chat.py b/app/llm/external/openai_chat.py index dde7d3f0..dfe5711f 100644 --- a/app/llm/external/openai_chat.py +++ b/app/llm/external/openai_chat.py @@ -1,3 +1,5 @@ +import logging +import time from datetime import datetime from typing import Literal, Any @@ -78,16 +80,34 @@ def chat( self, messages: list[PyrisMessage], arguments: CompletionArguments ) -> PyrisMessage: # noinspection PyTypeChecker - response = self._client.chat.completions.create( - model=self.model, - messages=convert_to_open_ai_messages(messages), - temperature=arguments.temperature, - max_tokens=arguments.max_tokens, - response_format=ResponseFormat( - type=("json_object" if arguments.response_format == "JSON" else "text") - ), - ) - return convert_to_iris_message(response.choices[0].message) + retries = 10 + backoff_factor = 2 + initial_delay = 1 + + for attempt in range(retries): + try: + if arguments.response_format == "JSON": + response = self._client.chat.completions.create( + model=self.model, + messages=convert_to_open_ai_messages(messages), + temperature=arguments.temperature, + max_tokens=arguments.max_tokens, + response_format=ResponseFormat(type="json_object"), + ) + else: + response = self._client.chat.completions.create( + model=self.model, + messages=convert_to_open_ai_messages(messages), + temperature=arguments.temperature, + max_tokens=arguments.max_tokens, + ) + return convert_to_iris_message(response.choices[0].message) + except Exception as e: + wait_time = initial_delay * (backoff_factor**attempt) + logging.warning(f"Exception on attempt {attempt + 1}: {e}") + logging.info(f"Retrying in {wait_time} seconds...") + time.sleep(wait_time) + logging.error("Failed to interpret image after several attempts.") class DirectOpenAIChatModel(OpenAIChatModel): diff --git a/app/llm/external/openai_embeddings.py b/app/llm/external/openai_embeddings.py index 6f7b19ad..243860df 100644 --- a/app/llm/external/openai_embeddings.py +++ b/app/llm/external/openai_embeddings.py @@ -1,8 +1,10 @@ +import logging from typing import Literal, Any from openai import OpenAI from openai.lib.azure import AzureOpenAI from ...llm.external.model import EmbeddingModel +import time class OpenAIEmbeddingModel(EmbeddingModel): @@ -11,12 +13,27 @@ class OpenAIEmbeddingModel(EmbeddingModel): _client: OpenAI def embed(self, text: str) -> list[float]: - response = self._client.embeddings.create( - model=self.model, - input=text, - encoding_format="float", + retries = 10 + backoff_factor = 2 + initial_delay = 1 + + for attempt in range(retries): + try: + response = self._client.embeddings.create( + model=self.model, + input=text, + encoding_format="float", + ) + return response.data[0].embedding + except Exception as e: + wait_time = initial_delay * (backoff_factor**attempt) + logging.warning(f"Rate limit exceeded on attempt {attempt + 1}: {e}") + logging.info(f"Retrying in {wait_time} seconds...") + time.sleep(wait_time) + logging.error( + "Failed to get embedding after several attempts due to rate limit." ) - return response.data[0].embedding + return [] class DirectOpenAIEmbeddingModel(OpenAIEmbeddingModel): diff --git a/app/llm/request_handler/__init__.py b/app/llm/request_handler/__init__.py index d43e448b..ab02e05a 100644 --- a/app/llm/request_handler/__init__.py +++ b/app/llm/request_handler/__init__.py @@ -1,5 +1,6 @@ from ..request_handler.request_handler_interface import RequestHandler from ..request_handler.basic_request_handler import BasicRequestHandler + from ..request_handler.capability_request_handler import ( CapabilityRequestHandler, CapabilityRequestHandlerSelectionMode, diff --git a/app/pipeline/chat/lecture_chat_pipeline.py b/app/pipeline/chat/lecture_chat_pipeline.py index 80be7e95..720364d6 100644 --- a/app/pipeline/chat/lecture_chat_pipeline.py +++ b/app/pipeline/chat/lecture_chat_pipeline.py @@ -8,6 +8,7 @@ ) from langchain_core.runnables import Runnable +from ..shared.citation_pipeline import CitationPipeline from ...common import convert_iris_message_to_langchain_message from ...domain import PyrisMessage from ...llm import CapabilityRequestHandler, RequirementList @@ -42,7 +43,8 @@ def lecture_initial_prompt(): questions about the lectures. To answer them the best way, relevant lecture content is provided to you with the student's question. If the context provided to you is not enough to formulate an answer to the student question you can simply ask the student to elaborate more on his question. Use only the parts of the context provided for - you that is relevant to the student's question. """ + you that is relevant to the student's question. If the user greets you greet him back, and ask him how you can help + """ class LectureChatPipeline(Pipeline): @@ -60,7 +62,7 @@ def __init__(self): privacy_compliance=True, ) ) - completion_args = CompletionArguments(temperature=0.2, max_tokens=2000) + completion_args = CompletionArguments(temperature=0, max_tokens=2000) self.llm = IrisLangchainChatModel( request_handler=request_handler, completion_args=completion_args ) @@ -68,6 +70,7 @@ def __init__(self): self.db = VectorDatabase() self.retriever = LectureRetrieval(self.db.client) self.pipeline = self.llm | StrOutputParser() + self.citation_pipeline = CitationPipeline() def __repr__(self): return f"{self.__class__.__name__}(llm={self.llm})" @@ -98,6 +101,8 @@ def __call__(self, dto: LectureChatPipelineExecutionDTO): student_query=query.contents[0].text_content, result_limit=10, course_name=dto.course.name, + course_id=dto.course.id, + base_url=dto.settings.artemis_base_url, ) self._add_relevant_chunks_to_prompt(retrieved_lecture_chunks) @@ -105,8 +110,11 @@ def __call__(self, dto: LectureChatPipelineExecutionDTO): self.prompt = ChatPromptTemplate.from_messages(prompt_val) try: response = (self.prompt | self.pipeline).invoke({}) + response_with_citation = self.citation_pipeline( + retrieved_lecture_chunks, response + ) logger.info(f"Response from lecture chat pipeline: {response}") - return response + return response_with_citation except Exception as e: raise e diff --git a/app/pipeline/chat/tutor_chat_pipeline.py b/app/pipeline/chat/tutor_chat_pipeline.py index 4fe4371d..2e4863fc 100644 --- a/app/pipeline/chat/tutor_chat_pipeline.py +++ b/app/pipeline/chat/tutor_chat_pipeline.py @@ -12,6 +12,7 @@ PromptTemplate, ) from langchain_core.runnables import Runnable +from weaviate.collections.classes.filters import Filter from .lecture_chat_pipeline import LectureChatPipeline from .output_models.output_models.selected_paragraphs import SelectedParagraphs @@ -86,19 +87,27 @@ def __call__(self, dto: TutorChatPipelineExecutionDTO, **kwargs): :param dto: execution data transfer object :param kwargs: The keyword arguments """ - execution_dto = LectureChatPipelineExecutionDTO( - settings=dto.settings, course=dto.course, chatHistory=dto.chat_history - ) - lecture_chat_thread = threading.Thread( - target=self._run_lecture_chat_pipeline(execution_dto), args=(dto,) - ) - tutor_chat_thread = threading.Thread( - target=self._run_tutor_chat_pipeline(dto), args=(dto,) - ) - lecture_chat_thread.start() - tutor_chat_thread.start() - try: + should_execute_lecture_pipeline = self.should_execute_lecture_pipeline( + dto.course.id + ) + self.lecture_chat_response = "" + if should_execute_lecture_pipeline: + execution_dto = LectureChatPipelineExecutionDTO( + settings=dto.settings, + course=dto.course, + chatHistory=dto.chat_history, + ) + lecture_chat_thread = threading.Thread( + target=self._run_lecture_chat_pipeline(execution_dto), args=(dto,) + ) + lecture_chat_thread.start() + + tutor_chat_thread = threading.Thread( + target=self._run_tutor_chat_pipeline(dto), + args=(dto, should_execute_lecture_pipeline), + ) + tutor_chat_thread.start() response = self.choose_best_response( [self.tutor_chat_response, self.lecture_chat_response], dto.chat_history[-1].contents[0].text_content, @@ -107,7 +116,6 @@ def __call__(self, dto: TutorChatPipelineExecutionDTO, **kwargs): logger.info(f"Response from tutor chat pipeline: {response}") self.callback.done("Generated response", final_result=response) except Exception as e: - print(e) self.callback.error(f"Failed to generate response: {e}") def choose_best_response( @@ -152,7 +160,11 @@ def _run_lecture_chat_pipeline(self, dto: LectureChatPipelineExecutionDTO): pipeline = LectureChatPipeline() self.lecture_chat_response = pipeline(dto=dto) - def _run_tutor_chat_pipeline(self, dto: TutorChatPipelineExecutionDTO): + def _run_tutor_chat_pipeline( + self, + dto: TutorChatPipelineExecutionDTO, + should_execute_lecture_pipeline: bool = False, + ): """ Runs the pipeline :param dto: execution data transfer object @@ -208,16 +220,18 @@ def _run_tutor_chat_pipeline(self, dto: TutorChatPipelineExecutionDTO): submission, selected_files, ) - - retrieved_lecture_chunks = self.retriever( - chat_history=history, - student_query=query.contents[0].text_content, - result_limit=10, - course_name=dto.course.name, - problem_statement=problem_statement, - exercise_title=exercise_title, - ) - self._add_relevant_chunks_to_prompt(retrieved_lecture_chunks) + if should_execute_lecture_pipeline: + retrieved_lecture_chunks = self.retriever( + chat_history=history, + student_query=query.contents[0].text_content, + result_limit=5, + course_name=dto.course.name, + problem_statement=problem_statement, + exercise_title=exercise_title, + course_id=dto.course.id, + base_url=dto.settings.artemis_base_url, + ) + self._add_relevant_chunks_to_prompt(retrieved_lecture_chunks) self.callback.in_progress("Generating response...") @@ -360,3 +374,21 @@ def _add_relevant_chunks_to_prompt(self, retrieved_lecture_chunks: List[dict]): self.prompt += SystemMessagePromptTemplate.from_template( "USE ONLY THE CONTENT YOU NEED TO ANSWER THE QUESTION:\n" ) + + def should_execute_lecture_pipeline(self, course_id: int) -> bool: + """ + Checks if the lecture pipeline should be executed + :param course_id: The course ID + :return: True if the lecture pipeline should be executed + """ + if course_id: + # Fetch the first object that matches the course ID with the language property + result = self.db.lectures.query.fetch_objects( + filters=Filter.by_property(LectureSchema.COURSE_ID.value).equal( + course_id + ), + limit=1, + return_properties=[LectureSchema.COURSE_NAME.value], + ) + return len(result.objects) > 0 + return False diff --git a/app/pipeline/lecture_ingestion_pipeline.py b/app/pipeline/lecture_ingestion_pipeline.py new file mode 100644 index 00000000..68ca02de --- /dev/null +++ b/app/pipeline/lecture_ingestion_pipeline.py @@ -0,0 +1,321 @@ +import base64 +import os +import tempfile +import threading +from asyncio.log import logger +import fitz +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate +from unstructured.cleaners.core import clean +from weaviate import WeaviateClient +from weaviate.classes.query import Filter +from . import Pipeline +from ..domain import IrisMessageRole, PyrisMessage +from ..domain.data.image_message_content_dto import ImageMessageContentDTO + +from ..domain.data.lecture_unit_dto import LectureUnitDTO +from app.domain.ingestion.ingestion_pipeline_execution_dto import ( + IngestionPipelineExecutionDto, +) +from ..domain.data.text_message_content_dto import TextMessageContentDTO +from ..llm.langchain import IrisLangchainChatModel +from ..vector_database.lecture_schema import init_lecture_schema, LectureSchema +from ..ingestion.abstract_ingestion import AbstractIngestion +from ..llm import ( + BasicRequestHandler, + CompletionArguments, + CapabilityRequestHandler, + RequirementList, +) +from ..web.status import IngestionStatusCallback +from langchain_text_splitters import RecursiveCharacterTextSplitter + +batch_update_lock = threading.Lock() + + +def cleanup_temporary_file(file_path): + """ + Cleanup the temporary file + """ + try: + os.remove(file_path) + except OSError as e: + logger.error(f"Failed to remove temporary file {file_path}: {e}") + + +def save_pdf(pdf_file_base64): + """ + Save the pdf file to a temporary file + """ + binary_data = base64.b64decode(pdf_file_base64) + fd, temp_pdf_file_path = tempfile.mkstemp(suffix=".pdf") + os.close(fd) + with open(temp_pdf_file_path, "wb") as temp_pdf_file: + try: + temp_pdf_file.write(binary_data) + except Exception as e: + logger.error( + f"Failed to write to temporary PDF file {temp_pdf_file_path}: {e}" + ) + raise + return temp_pdf_file_path + + +def create_page_data(page_num, page_splits, lecture_unit_dto, course_language): + """ + Create and return a list of dictionnaries to be ingested in the Vector Database. + """ + return [ + { + LectureSchema.LECTURE_ID.value: lecture_unit_dto.lecture_id, + LectureSchema.LECTURE_NAME.value: lecture_unit_dto.lecture_name, + LectureSchema.LECTURE_UNIT_ID.value: lecture_unit_dto.lecture_unit_id, + LectureSchema.LECTURE_UNIT_NAME.value: lecture_unit_dto.lecture_unit_name, + LectureSchema.COURSE_ID.value: lecture_unit_dto.course_id, + LectureSchema.COURSE_NAME.value: lecture_unit_dto.course_name, + LectureSchema.COURSE_DESCRIPTION.value: lecture_unit_dto.course_description, + LectureSchema.BASE_URL.value: lecture_unit_dto.base_url, + LectureSchema.COURSE_LANGUAGE.value: course_language, + LectureSchema.PAGE_NUMBER.value: page_num + 1, + LectureSchema.PAGE_TEXT_CONTENT.value: page_split.page_content, + } + for page_split in page_splits + ] + + +class LectureIngestionPipeline(AbstractIngestion, Pipeline): + + def __init__( + self, + client: WeaviateClient, + dto: IngestionPipelineExecutionDto, + callback: IngestionStatusCallback, + ): + super().__init__() + self.collection = init_lecture_schema(client) + self.dto = dto + self.llm_vision = BasicRequestHandler("azure-gpt-4-vision") + self.llm_chat = BasicRequestHandler( + "azure-gpt-35-turbo" + ) # TODO change use langain model + self.llm_embedding = BasicRequestHandler("embedding-small") + self.callback = callback + request_handler = CapabilityRequestHandler( + requirements=RequirementList( + gpt_version_equivalent=3.5, + context_length=16385, + privacy_compliance=True, + ) + ) + completion_args = CompletionArguments(temperature=0.2, max_tokens=2000) + self.llm = IrisLangchainChatModel( + request_handler=request_handler, completion_args=completion_args + ) + self.pipeline = self.llm | StrOutputParser() + + def __call__(self) -> bool: + try: + self.callback.in_progress("Deleting old slides from database...") + self.delete_old_lectures() + self.callback.done("Old slides removed") + # Here we check if the operation is for updating or for deleting, + # we only check the first file because all the files will have the same operation + if not self.dto.lecture_units[0].to_update: + self.callback.skip("Lecture Chunking and interpretation Skipped") + self.callback.skip("No new slides to update") + return True + self.callback.in_progress("Chunking and interpreting lecture...") + chunks = [] + for i, lecture_unit in enumerate(self.dto.lecture_units): + pdf_path = save_pdf(lecture_unit.pdf_file_base64) + chunks.extend( + self.chunk_data(lecture_pdf=pdf_path, lecture_unit_dto=lecture_unit) + ) + cleanup_temporary_file(pdf_path) + self.callback.done("Lecture Chunking and interpretation Finished") + self.callback.in_progress("Ingesting lecture chunks into database...") + self.batch_update(chunks) + self.callback.done("Lecture Ingestion Finished") + logger.info( + f"Lecture ingestion pipeline finished Successfully for course " + f"{self.dto.lecture_units[0].course_name}" + ) + return True + except Exception as e: + logger.error(f"Error updating lecture unit: {e}") + self.callback.error(f"Failed to ingest lectures into the database: {e}") + return False + + def batch_update(self, chunks): + """ + Batch update the chunks into the database + This method is thread-safe and can only be executed by one thread at a time. + Weaviate limitation. + """ + global batch_update_lock + with batch_update_lock: + with self.collection.batch.rate_limit(requests_per_minute=600) as batch: + try: + for index, chunk in enumerate(chunks): + embed_chunk = self.llm_embedding.embed( + chunk[LectureSchema.PAGE_TEXT_CONTENT.value] + ) + batch.add_object(properties=chunk, vector=embed_chunk) + except Exception as e: + logger.error(f"Error updating lecture unit: {e}") + self.callback.error( + f"Failed to ingest lectures into the database: {e}" + ) + + def chunk_data( + self, + lecture_pdf: str, + lecture_unit_dto: LectureUnitDTO = None, + ): + """ + Chunk the data from the lecture into smaller pieces + """ + doc = fitz.open(lecture_pdf) + course_language = self.get_course_language( + doc.load_page(min(5, doc.page_count - 1)).get_text() + ) + data = [] + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=512, chunk_overlap=102 + ) + for page_num in range(doc.page_count): + page = doc.load_page(page_num) + page_text = page.get_text() + if page.get_images(full=False): + # more pixels thus more details and better quality + matrix = fitz.Matrix(20.0, 20.0) + pix = page.get_pixmap(matrix=matrix) + img_bytes = pix.tobytes("jpg") + img_base64 = base64.b64encode(img_bytes).decode("utf-8") + image_interpretation = self.interpret_image( + img_base64, + page_text, + lecture_unit_dto.lecture_name, + course_language, + ) + page_text = self.merge_page_content_and_image_interpretation( + page_text, image_interpretation + ) + page_splits = text_splitter.create_documents([page_text]) + data.extend( + create_page_data( + page_num, page_splits, lecture_unit_dto, course_language + ) + ) + return data + + def interpret_image( + self, + img_base64: str, + last_page_content: str, + name_of_lecture: str, + course_language: str, + ): + """ + Interpret the image passed + """ + image_interpretation_prompt = TextMessageContentDTO( + text_content=f"This page is part of the {name_of_lecture} university lecture," + f" explain what is on the slide in an academic way," + f" respond only with the explanation in {course_language}." + f" For more context here is the content of the previous slide: " + f" {last_page_content}" + ) + image = ImageMessageContentDTO(base64=img_base64) + iris_message = PyrisMessage( + sender=IrisMessageRole.USER, contents=[image_interpretation_prompt, image] + ) + try: + response = self.llm_vision.chat( + [iris_message], CompletionArguments(temperature=0, max_tokens=400) + ) + except Exception as e: + logger.error(f"Error interpreting image: {e}") + return None + return response.contents[0].text_content + + def merge_page_content_and_image_interpretation( + self, page_content: str, image_interpretation: str + ): + """ + Merge the text and image together + """ + dirname = os.path.dirname(__file__) + prompt_file_path = os.path.join( + dirname, ".", "prompts", "content_image_interpretation_merge_prompt.txt" + ) + with open(prompt_file_path, "r") as file: + logger.info("Loading ingestion prompt...") + lecture_ingestion_prompt = file.read() + prompt = ChatPromptTemplate.from_messages( + [ + ("system", lecture_ingestion_prompt), + ] + ) + prompt_val = prompt.format_messages( + page_content=page_content, + image_interpretation=image_interpretation, + ) + prompt = ChatPromptTemplate.from_messages(prompt_val) + return clean( + (prompt | self.pipeline).invoke({}), bullets=True, extra_whitespace=True + ) + + def get_course_language(self, page_content: str) -> str: + """ + Translate the student query to the course language. For better retrieval. + """ + prompt = ( + f"You will be provided a chunk of text, respond with the language of the text. Do not respond with " + f"anything else than the language.\nHere is the text: \n{page_content}" + ) + iris_message = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[TextMessageContentDTO(text_content=prompt)], + ) + response = self.llm_chat.chat( + [iris_message], CompletionArguments(temperature=0, max_tokens=20) + ) + return response.contents[0].text_content + + def delete_old_lectures(self): + """ + Delete the lecture unit from the database + """ + try: + for lecture_unit in self.dto.lecture_units: + if self.delete_lecture_unit( + lecture_unit.course_id, + lecture_unit.lecture_id, + lecture_unit.lecture_unit_id, + lecture_unit.base_url, + ): + logger.info("Lecture deleted successfully") + else: + logger.error("Failed to delete lecture") + except Exception as e: + logger.error(f"Error deleting lecture unit: {e}") + return False + + def delete_lecture_unit(self, course_id, lecture_id, lecture_unit_id, base_url): + """ + Delete the lecture from the database + """ + try: + self.collection.data.delete_many( + where=Filter.by_property(LectureSchema.BASE_URL.value).equal(base_url) + & Filter.by_property(LectureSchema.COURSE_ID.value).equal(course_id) + & Filter.by_property(LectureSchema.LECTURE_ID.value).equal(lecture_id) + & Filter.by_property(LectureSchema.LECTURE_UNIT_ID.value).equal( + lecture_unit_id + ) + ) + return True + except Exception as e: + logger.error(f"Error deleting lecture unit: {e}", exc_info=True) + return False diff --git a/app/pipeline/prompts/choose_response_prompt.txt b/app/pipeline/prompts/choose_response_prompt.txt index 78779f2d..704c1455 100644 --- a/app/pipeline/prompts/choose_response_prompt.txt +++ b/app/pipeline/prompts/choose_response_prompt.txt @@ -8,6 +8,7 @@ unnecessary information, only the number of the paragraph that is most relevant If the question is asking for code, return {{"selected_paragraphs": [0]}} Do not by any means return a the number of the response that has written programming code in it. If there is no suitable answer return {{"selected_paragraphs": [0]}} +If the question is a type of greeting like hello or hey return {{"selected_paragraphs": [0]}} If the answer or the question is out the education context return {{"selected_paragraphs": [0]}} Paragraph 0: diff --git a/app/pipeline/prompts/citation_prompt.txt b/app/pipeline/prompts/citation_prompt.txt new file mode 100644 index 00000000..1c2b2fc0 --- /dev/null +++ b/app/pipeline/prompts/citation_prompt.txt @@ -0,0 +1,18 @@ +In the paragraphs below you are provided with an answer to a question. Underneath the answer you will find the paragraphs that the answer was based on. +Add citations of the paragraphs to the answer. Cite the paragraphs in brackets after the sentence where the information is used in the answer. +At the end of the answer list each source with its corresponding number and provide the Lecture Title,as well as the page number in this format "[1] Lecture title, page number". +Do not Include the Actual paragraphs, only the citations at the end. +If the question is not a question, or is a greeting, do not add any citations. +Here is an example how to rewrite the answer with citations: +" +Lorem ipsum dolor sit amet, consectetur adipiscing elit.[1] Ded do eiusmod tempor incididunt ut labore et dolore magna aliqua.[2] + +[1] Lecture 1, page 2. +[2] Lecture 2, page 5. +" + + +Here are the answer and the paragraphs: + +Answer without citations: +{Answer} diff --git a/app/pipeline/prompts/content_image_interpretation_merge_prompt.txt b/app/pipeline/prompts/content_image_interpretation_merge_prompt.txt new file mode 100644 index 00000000..cded68f9 --- /dev/null +++ b/app/pipeline/prompts/content_image_interpretation_merge_prompt.txt @@ -0,0 +1,19 @@ +You are An AI assistant for university Professors of the Technical University of Munich. +You are tasked with helping to prepare educational materials for university students. +You were provided with the raw text content of a slide and, in some cases, + a description of the slide generated by another AI assistant. +The assistant can fail to generate a description for some slides. +Your task is to merge the description and the text content of the slide. +If a description is available, you should add it after the raw text content of the slide. +If an error message is given at the description, please ignore it and return only the raw text content. + + +############################################################################################################ +Here is the raw text content of the Slide provided: +{page_content} +############################################################################################################ + +############################################################################################################ +Here is the description of the slide provided, if it's an error message ignore it: +{image_interpretation} +############################################################################################################ diff --git a/app/pipeline/prompts/lecture_retrieval_prompts.py b/app/pipeline/prompts/lecture_retrieval_prompts.py index 69d8f6bf..8fb55857 100644 --- a/app/pipeline/prompts/lecture_retrieval_prompts.py +++ b/app/pipeline/prompts/lecture_retrieval_prompts.py @@ -30,7 +30,7 @@ write_hypothetical_answer_prompt = """ Please provide a response in {course_language}. Craft your response to closely reflect the style and content of university lecture materials. - Do not exceed 500 characters. + Do not exceed 300 words. Add keywords and phrases that are relevant to student intent. """ diff --git a/app/pipeline/shared/citation_pipeline.py b/app/pipeline/shared/citation_pipeline.py new file mode 100644 index 00000000..b547d315 --- /dev/null +++ b/app/pipeline/shared/citation_pipeline.py @@ -0,0 +1,94 @@ +import os +from asyncio.log import logger +from typing import Optional, List, Union + +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate, PromptTemplate +from langchain_core.runnables import Runnable + +from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments +from app.llm.langchain import IrisLangchainChatModel +from app.pipeline import Pipeline + +from app.vector_database.lecture_schema import LectureSchema + + +class CitationPipeline(Pipeline): + """A generic reranker pipeline that can be used to rerank a list of documents based on a question""" + + llm: IrisLangchainChatModel + pipeline: Runnable + prompt_str: str + prompt: ChatPromptTemplate + + def __init__(self): + super().__init__(implementation_id="citation_pipeline") + request_handler = CapabilityRequestHandler( + requirements=RequirementList( + gpt_version_equivalent=3.5, + context_length=16385, + ) + ) + self.llm = IrisLangchainChatModel( + request_handler=request_handler, + completion_args=CompletionArguments(temperature=0, max_tokens=4000), + ) + dirname = os.path.dirname(__file__) + prompt_file_path = os.path.join(dirname, "..", "prompts", "citation_prompt.txt") + with open(prompt_file_path, "r") as file: + self.prompt_str = file.read() + self.pipeline = self.llm | StrOutputParser() + + def __repr__(self): + return f"{self.__class__.__name__}(llm={self.llm})" + + def __str__(self): + return f"{self.__class__.__name__}(llm={self.llm})" + + def create_formatted_string(self, paragraphs): + """ + Create a formatted string from the data + """ + formatted_string = "" + for i, paragraph in enumerate(paragraphs): + para = paragraph.get(LectureSchema.PAGE_TEXT_CONTENT.value, "") + title = "Lecture Title:" + paragraph.get( + LectureSchema.LECTURE_NAME.value, "" + ) + page_number = paragraph.get(LectureSchema.PAGE_NUMBER.value, "") + formatted_string += ( + "-" * 50 + "\n\n" + f"{title}page {page_number}:" + f"\n\n{para}\n" + "-" * 50 + "\n\n" + ) + + return ( + formatted_string.replace("{", "{{").replace("}", "}}") + + "\n Answer with citations: \n" + ) + + def __call__( + self, + paragraphs: Union[List[dict], List[str]], + answer: str, + prompt: Optional[PromptTemplate] = None, + **kwargs, + ) -> List[str]: + """ + Runs the pipeline + :param paragraphs: List of paragraphs which can be list of dicts or list of strings + :param query: The query + :return: Selected file content + """ + self.prompt_str += "\n" + self.create_formatted_string(paragraphs) + + try: + self.default_prompt = PromptTemplate( + template=self.prompt_str, + input_variables=["Answer"], + ) + response = (self.default_prompt | self.pipeline).invoke({"Answer": answer}) + return response + except Exception as e: + logger.error("citation pipeline failed", e) + raise e diff --git a/app/retrieval/lecture_retrieval.py b/app/retrieval/lecture_retrieval.py index 0d5c567e..c68c5442 100644 --- a/app/retrieval/lecture_retrieval.py +++ b/app/retrieval/lecture_retrieval.py @@ -88,7 +88,7 @@ def __init__(self, client: WeaviateClient, **kwargs): privacy_compliance=True, ) ) - completion_args = CompletionArguments(temperature=0.2, max_tokens=2000) + completion_args = CompletionArguments(temperature=0, max_tokens=2000) self.llm = IrisLangchainChatModel( request_handler=request_handler, completion_args=completion_args ) @@ -104,21 +104,15 @@ def __call__( result_limit: int, course_name: str = None, course_id: int = None, + base_url: str = None, problem_statement: str = None, exercise_title: str = None, ) -> List[dict]: """ Retrieve lecture data from the database. """ - course_language = ( - self.collection.query.fetch_objects( - limit=1, return_properties=[LectureSchema.COURSE_LANGUAGE.value] - ) - .objects[0] - .properties.get(LectureSchema.COURSE_LANGUAGE.value) - ) + course_language = self.fetch_course_language(course_id) - # Call the function to run the tasks response, response_hyde = self.run_parallel_rewrite_tasks( chat_history=chat_history, student_query=student_query, @@ -126,6 +120,7 @@ def __call__( course_language=course_language, course_name=course_name, course_id=course_id, + base_url=base_url, problem_statement=problem_statement, exercise_title=exercise_title, ) @@ -141,11 +136,12 @@ def __call__( merged_chunks = merge_retrieved_chunks( basic_retrieved_lecture_chunks, hyde_retrieved_lecture_chunks ) - - selected_chunks_index = self.reranker_pipeline( - paragraphs=merged_chunks, query=student_query, chat_history=chat_history - ) - return [merged_chunks[int(i)] for i in selected_chunks_index] + if len(merged_chunks) != 0: + selected_chunks_index = self.reranker_pipeline( + paragraphs=merged_chunks, query=student_query, chat_history=chat_history + ) + return [merged_chunks[int(i)] for i in selected_chunks_index] + return [] def rewrite_student_query( self, @@ -290,29 +286,51 @@ def rewrite_elaborated_query_with_exercise_context( raise e def search_in_db( - self, query: str, hybrid_factor: float, result_limit: int, course_id: int = None + self, + query: str, + hybrid_factor: float, + result_limit: int, + course_id: int = None, + base_url: str = None, ): """ - Search the query in the database and return the results. + Search the database for the given query. """ - return self.collection.query.hybrid( + # Initialize filter to None by default + filter_weaviate = None + + # Check if course_id is provided + if course_id: + # Create a filter for course_id + filter_weaviate = Filter.by_property(LectureSchema.COURSE_ID.value).equal( + course_id + ) + + # Extend the filter based on the presence of base_url + if base_url: + filter_weaviate &= Filter.by_property( + LectureSchema.BASE_URL.value + ).equal(base_url) + else: + filter_weaviate = Filter.by_property( + LectureSchema.BASE_URL.value + ).equal(base_url) + + return_value = self.collection.query.hybrid( query=query, - filters=( - Filter.by_property(LectureSchema.COURSE_ID.value).equal(course_id) - if course_id - else None - ), alpha=hybrid_factor, vector=self.llm_embedding.embed(query), return_properties=[ LectureSchema.PAGE_TEXT_CONTENT.value, - LectureSchema.PAGE_IMAGE_DESCRIPTION.value, LectureSchema.COURSE_NAME.value, LectureSchema.LECTURE_NAME.value, LectureSchema.PAGE_NUMBER.value, + LectureSchema.COURSE_ID.value, ], limit=result_limit, + filters=filter_weaviate, ) + return return_value def run_parallel_rewrite_tasks( self, @@ -322,6 +340,7 @@ def run_parallel_rewrite_tasks( course_language: str, course_name: str = None, course_id: int = None, + base_url: str = None, problem_statement: str = None, exercise_title: str = None, ): @@ -351,8 +370,10 @@ def run_parallel_rewrite_tasks( ) # Get the results once both tasks are complete - rewritten_query = rewritten_query_future.result() - hypothetical_answer_query = hypothetical_answer_query_future.result() + rewritten_query: str = rewritten_query_future.result() + hypothetical_answer_query: str = ( + hypothetical_answer_query_future.result() + ) else: with concurrent.futures.ThreadPoolExecutor() as executor: # Schedule the rewrite tasks to run in parallel @@ -378,10 +399,20 @@ def run_parallel_rewrite_tasks( # Execute the database search tasks with concurrent.futures.ThreadPoolExecutor() as executor: response_future = executor.submit( - self.search_in_db, rewritten_query, 1, result_limit, course_id + self.search_in_db, + query=rewritten_query, + hybrid_factor=0.7, + result_limit=result_limit, + course_id=course_id, + base_url=base_url, ) response_hyde_future = executor.submit( - self.search_in_db, hypothetical_answer_query, 1, result_limit, course_id + self.search_in_db, + query=hypothetical_answer_query, + hybrid_factor=0.9, + result_limit=result_limit, + course_id=course_id, + base_url=base_url, ) # Get the results once both tasks are complete @@ -389,3 +420,30 @@ def run_parallel_rewrite_tasks( response_hyde = response_hyde_future.result() return response, response_hyde + + def fetch_course_language(self, course_id): + """ + Fetch the language of the course based on the course ID. + If no specific language is set, it defaults to English. + """ + course_language = "english" + + if course_id: + # Fetch the first object that matches the course ID with the language property + result = self.collection.query.fetch_objects( + filters=Filter.by_property(LectureSchema.COURSE_ID.value).equal( + course_id + ), + limit=1, # We only need one object to check and retrieve the language + return_properties=[LectureSchema.COURSE_LANGUAGE.value], + ) + + # Check if the result has objects and retrieve the language + if result.objects: + fetched_language = result.objects[0].properties.get( + LectureSchema.COURSE_LANGUAGE.value + ) + if fetched_language: + course_language = fetched_language + + return course_language diff --git a/app/vector_database/__init__.py b/app/vector_database/__init__.py index e69de29b..a1858065 100644 --- a/app/vector_database/__init__.py +++ b/app/vector_database/__init__.py @@ -0,0 +1,3 @@ +import app.vector_database.database +import app.vector_database.lecture_schema +import app.vector_database.repository_schema diff --git a/app/vector_database/database.py b/app/vector_database/database.py index f670c372..52bde3b0 100644 --- a/app/vector_database/database.py +++ b/app/vector_database/database.py @@ -1,9 +1,8 @@ import logging -import os import weaviate from .lecture_schema import init_lecture_schema -from .repository_schema import init_repository_schema -import weaviate.classes as wvc +from weaviate.classes.query import Filter +from app.config import settings logger = logging.getLogger(__name__) @@ -14,11 +13,11 @@ class VectorDatabase: """ def __init__(self): - self.client = weaviate.connect_to_wcs( - cluster_url=os.getenv("WEAVIATE_CLUSTER_URL"), - auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_AUTH_KEY")), + self.client = weaviate.connect_to_local( + host=settings.weaviate.host, + port=settings.weaviate.port, + grpc_port=settings.weaviate.grpc_port, ) - self.repositories = init_repository_schema(self.client) self.lectures = init_lecture_schema(self.client) def __del__(self): @@ -40,5 +39,11 @@ def delete_object(self, collection_name, property_name, object_property): """ collection = self.client.collections.get(collection_name) collection.data.delete_many( - where=wvc.query.Filter.by_property(property_name).equal(object_property) + where=Filter.by_property(property_name).equal(object_property) ) + + def get_client(self): + """ + Get the Weaviate client + """ + return self.client diff --git a/app/vector_database/lecture_schema.py b/app/vector_database/lecture_schema.py index 08e253a3..912abed7 100644 --- a/app/vector_database/lecture_schema.py +++ b/app/vector_database/lecture_schema.py @@ -21,9 +21,8 @@ class LectureSchema(Enum): LECTURE_UNIT_ID = "lecture_unit_id" LECTURE_UNIT_NAME = "lecture_unit_name" PAGE_TEXT_CONTENT = "page_text_content" - PAGE_IMAGE_DESCRIPTION = "page_image_explanation" - PAGE_BASE64 = "page_base64" PAGE_NUMBER = "page_number" + BASE_URL = "base_url" def init_lecture_schema(client: WeaviateClient) -> Collection: @@ -43,21 +42,25 @@ def init_lecture_schema(client: WeaviateClient) -> Collection: name=LectureSchema.COURSE_ID.value, description="The ID of the course", data_type=DataType.INT, + index_searchable=False, ), Property( name=LectureSchema.COURSE_NAME.value, description="The name of the course", data_type=DataType.TEXT, + index_searchable=False, ), Property( name=LectureSchema.COURSE_DESCRIPTION.value, description="The description of the COURSE", data_type=DataType.TEXT, + index_searchable=False, ), Property( name=LectureSchema.LECTURE_ID.value, description="The ID of the lecture", data_type=DataType.INT, + index_searchable=False, ), Property( name=LectureSchema.LECTURE_NAME.value, @@ -68,6 +71,7 @@ def init_lecture_schema(client: WeaviateClient) -> Collection: name=LectureSchema.LECTURE_UNIT_ID.value, description="The ID of the lecture unit", data_type=DataType.INT, + index_searchable=False, ), Property( name=LectureSchema.LECTURE_UNIT_NAME.value, @@ -78,21 +82,19 @@ def init_lecture_schema(client: WeaviateClient) -> Collection: name=LectureSchema.PAGE_TEXT_CONTENT.value, description="The original text content from the slide", data_type=DataType.TEXT, - ), - Property( - name=LectureSchema.PAGE_IMAGE_DESCRIPTION.value, - description="The description of the slide if the slide contains an image", - data_type=DataType.TEXT, - ), - Property( - name=LectureSchema.PAGE_BASE64.value, - description="The base64 encoded image of the slide if the slide contains an image", - data_type=DataType.TEXT, + index_searchable=False, ), Property( name=LectureSchema.PAGE_NUMBER.value, description="The page number of the slide", data_type=DataType.INT, + index_searchable=False, + ), + Property( + name=LectureSchema.BASE_URL.value, + description="The base url of the website where the lecture slides are hosted", + data_type=DataType.TEXT, + index_searchable=False, ), ], ) diff --git a/app/web/routers/webhooks.py b/app/web/routers/webhooks.py index 66af9f8e..d269be5e 100644 --- a/app/web/routers/webhooks.py +++ b/app/web/routers/webhooks.py @@ -1,13 +1,53 @@ -from fastapi import APIRouter, status, Response +import traceback +from asyncio.log import logger +from threading import Thread, Semaphore + +from fastapi import APIRouter, status, Depends +from app.dependencies import TokenValidator +from app.domain.ingestion.ingestion_pipeline_execution_dto import ( + IngestionPipelineExecutionDto, +) +from ..status.IngestionStatusCallback import IngestionStatusCallback +from ...pipeline.lecture_ingestion_pipeline import LectureIngestionPipeline +from ...vector_database.database import VectorDatabase router = APIRouter(prefix="/api/v1/webhooks", tags=["webhooks"]) -@router.post("/lecture") -def lecture_webhook(): - return Response(status_code=status.HTTP_501_NOT_IMPLEMENTED) +semaphore = Semaphore(5) + + +def run_lecture_update_pipeline_worker(dto: IngestionPipelineExecutionDto): + """ + Run the tutor chat pipeline in a separate thread""" + with semaphore: + try: + callback = IngestionStatusCallback( + run_id=dto.settings.authentication_token, + base_url=dto.settings.artemis_base_url, + initial_stages=dto.initial_stages, + ) + db = VectorDatabase() + client = db.get_client() + pipeline = LectureIngestionPipeline( + client=client, dto=dto, callback=callback + ) + pipeline() + except Exception as e: + logger.error(f"Error Ingestion pipeline: {e}") + logger.error(traceback.format_exc()) + finally: + semaphore.release() -@router.post("/assignment") -def assignment_webhook(): - return Response(status_code=status.HTTP_501_NOT_IMPLEMENTED) +@router.post( + "/lectures/fullIngestion", + status_code=status.HTTP_202_ACCEPTED, + dependencies=[Depends(TokenValidator())], +) +def lecture_webhook(dto: IngestionPipelineExecutionDto): + """ + Webhook endpoint to trigger the tutor chat pipeline + """ + thread = Thread(target=run_lecture_update_pipeline_worker, args=(dto,)) + thread.start() diff --git a/app/web/status/IngestionStatusCallback.py b/app/web/status/IngestionStatusCallback.py new file mode 100644 index 00000000..a82a061c --- /dev/null +++ b/app/web/status/IngestionStatusCallback.py @@ -0,0 +1,41 @@ +from typing import List + +from .status_update import StatusCallback +from ...domain.ingestion.ingestion_status_update_dto import IngestionStatusUpdateDTO +from ...domain.status.stage_state_dto import StageStateEnum +from ...domain.status.stage_dto import StageDTO +import logging + +logger = logging.getLogger(__name__) + + +class IngestionStatusCallback(StatusCallback): + """ + Callback class for updating the status of a Tutor Chat pipeline run. + """ + + def __init__( + self, run_id: str, base_url: str, initial_stages: List[StageDTO] = None + ): + url = f"{base_url}/api/public/pyris/webhooks/ingestion/runs/{run_id}/status" + + current_stage_index = len(initial_stages) if initial_stages else 0 + stages = initial_stages or [] + stages += [ + StageDTO( + weight=10, state=StageStateEnum.NOT_STARTED, name="Old slides removal" + ), + StageDTO( + weight=60, + state=StageStateEnum.NOT_STARTED, + name="Slides Interpretation", + ), + StageDTO( + weight=30, + state=StageStateEnum.NOT_STARTED, + name="Slides ingestion", + ), + ] + status = IngestionStatusUpdateDTO(stages=stages) + stage = stages[current_stage_index] + super().__init__(url, run_id, status, stage, current_stage_index) diff --git a/docker/.docker-data/weaviate-data/.gitkeep b/docker/.docker-data/weaviate-data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker/pyris-dev.yml b/docker/pyris-dev.yml index 0d67e3ee..7d1a956d 100644 --- a/docker/pyris-dev.yml +++ b/docker/pyris-dev.yml @@ -15,7 +15,18 @@ services: networks: - pyris + weaviate: + extends: + file: ./weaviate.yml + service: weaviate + networks: + - pyris + ports: + - 8001:8001 + - 50051:50051 + networks: pyris: driver: "bridge" - name: pyris \ No newline at end of file + name: pyris + diff --git a/docker/pyris-production.yml b/docker/pyris-production.yml index 43400ddc..3329ae47 100644 --- a/docker/pyris-production.yml +++ b/docker/pyris-production.yml @@ -36,6 +36,13 @@ services: networks: - pyris + weaviate: + extends: + file: ./weaviate.yml + service: weaviate + networks: + - pyris + networks: pyris: driver: "bridge" diff --git a/docker/weaviate.yml b/docker/weaviate.yml new file mode 100644 index 00000000..d5103556 --- /dev/null +++ b/docker/weaviate.yml @@ -0,0 +1,19 @@ +--- +services: + weaviate: + command: + - --host + - 0.0.0.0 + - --port + - '8001' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:1.25.3 + expose: + - 8001 + - 50051 + volumes: + - ${WEAVIATE_VOLUME_MOUNT:-./.docker-data/weaviate-data}:/var/lib/weaviate + restart: on-failure:3 + env_file: + - ./weaviate/default.env \ No newline at end of file diff --git a/docker/weaviate/default.env b/docker/weaviate/default.env new file mode 100644 index 00000000..6a181fe7 --- /dev/null +++ b/docker/weaviate/default.env @@ -0,0 +1,10 @@ +QUERY_DEFAULTS_LIMIT=25 +AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true +PERSISTENCE_DATA_PATH=/var/lib/weaviate +DEFAULT_VECTORIZER_MODULE=none +ENABLE_MODULES= +CLUSTER_HOSTNAME=pyris +LIMIT_RESOURCES=true +DISK_USE_WARNING_PERCENTAGE=80 +vectorCacheMaxObjects=1000000 + diff --git a/example_application.yml b/example_application.yml new file mode 100644 index 00000000..5e3275ba --- /dev/null +++ b/example_application.yml @@ -0,0 +1,9 @@ +api_keys: + - token: "secret" + +weaviate: + host: "localhost" + port: "8001" + grpc_port: "50051" + +env_vars: diff --git a/requirements.txt b/requirements.txt index b0b3f302..c462b714 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ PyYAML==6.0.1 requests~=2.32.3 uvicorn==0.30.1 weaviate-client==4.6.4 +unstructured~=0.11.8 \ No newline at end of file