azure-ai-search-rag-eval-arize-ai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate RAG with LlamaIndex, Arize AI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install arize-phoenix\n",
    "!pip install azure-identity\n",
    "!pip install azure-search-documents==11.4.0\n",
    "!pip install llama-index\n",
    "!pip install llama-index-embeddings-azure-openai\n",
    "!pip install llama-index-llms-azure-openai\n",
    "!pip install llama-index-vector-stores-azureaisearch\n",
    "!pip install nest-asyncio\n",
    "!pip install \"openinference-instrumentation-llama-index>=2.0.0\"\n",
    "!pip install -U llama-index-callbacks-arize-phoenix\n",
    "!pip install python-dotenv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Setup\n",
    "Load environment variables and initialize the necessary clients and models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from llama_index.core import StorageContext, VectorStoreIndex\n",
    "from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding\n",
    "from llama_index.llms.azure_openai import AzureOpenAI\n",
    "from llama_index.vector_stores.azureaisearch import AzureAISearchVectorStore, IndexManagement\n",
    "\n",
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "# Environment Variables\n",
    "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "AZURE_OPENAI_API_KEY = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\") # I'm using GPT-3.5-turbo\n",
    "AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\") # I'm using text-embedding-ada-002\n",
    "SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "SEARCH_SERVICE_API_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "INDEX_NAME = \"contoso-hr-docs\"\n",
    "\n",
    "# Initialize Azure OpenAI and embedding models\n",
    "llm = AzureOpenAI(\n",
    "    model=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    deployment_name=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    api_version=\"2024-02-01\"\n",
    ")\n",
    "\n",
    "embed_model = AzureOpenAIEmbedding(\n",
    "    model=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME,\n",
    "    deployment_name=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    api_version=\"2024-02-01\"\n",
    ")\n",
    "\n",
    "# Initialize search clients\n",
    "credential = AzureKeyCredential(SEARCH_SERVICE_API_KEY)\n",
    "index_client = SearchIndexClient(endpoint=SEARCH_SERVICE_ENDPOINT, credential=credential)\n",
    "search_client = SearchClient(endpoint=SEARCH_SERVICE_ENDPOINT, index_name=INDEX_NAME, credential=credential)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Launch Phoenix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNI [phoenix.session.session] Existing running Phoenix instance detected! Shutting it down and starting a new instance...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🌍 To view the Phoenix app in your browser, visit http://localhost:6006/\n",
      "📖 For more information on how to use Phoenix, check out https://docs.arize.com/phoenix\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<phoenix.session.session.ThreadSession at 0x1b9c5336a50>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import phoenix as px\n",
    "import nest_asyncio\n",
    "import phoenix as px\n",
    "from llama_index.core import VectorStoreIndex\n",
    "import llama_index.core\n",
    "from llama_index.core.evaluation import AnswerRelevancyEvaluator, ContextRelevancyEvaluator\n",
    "from llama_index.core.llama_dataset import download_llama_dataset\n",
    "from llama_index.llms.openai import OpenAI\n",
    "from openinference.instrumentation.llama_index import LlamaIndexInstrumentor\n",
    "from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter\n",
    "from opentelemetry.sdk.trace import TracerProvider\n",
    "from opentelemetry.sdk.trace.export import SimpleSpanProcessor\n",
    "from phoenix.experiments import evaluate_experiment, run_experiment\n",
    "from phoenix.experiments.types import Explanation, Score\n",
    "\n",
    "nest_asyncio.apply()\n",
    "px.launch_app()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instrument LlamaIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import llama_index.core\n",
    "\n",
    "llama_index.core.set_global_handler(\"arize_phoenix\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vector Store Initialization\n",
    "Set up the vector store using Azure AI Search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.settings import Settings\n",
    "\n",
    "Settings.llm = llm\n",
    "Settings.embed_model = embed_model\n",
    "\n",
    "# Initialize the vector store\n",
    "vector_store = AzureAISearchVectorStore(\n",
    "    search_or_index_client=index_client,\n",
    "    index_name=INDEX_NAME,\n",
    "    index_management=IndexManagement.VALIDATE_INDEX,\n",
    "    id_field_key=\"id\",\n",
    "    chunk_field_key=\"text\",\n",
    "    embedding_field_key=\"embedding\",\n",
    "    embedding_dimensionality=1536,\n",
    "    metadata_string_field_key=\"metadata\",\n",
    "    doc_id_field_key=\"doc_id\",\n",
    "    language_analyzer=\"en.lucene\",\n",
    "    vector_algorithm_type=\"exhaustiveKnn\",\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Existing Index\n",
    "I'm going to use my existing \"contoso-hr-docs\" index that I created. For how to create an index and load documents from scratch, see [here](https://github.com/farzad528/azure-ai-search-python-playground/blob/addb1a29e70ee9dbf1bb9a39bbe367aa15e4cf5f/azure-ai-search-rag-eval-trulens.ipynb#L145)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    [],\n",
    "    storage_context=storage_context,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Query Execution\n",
    "Execute a query to test the setup."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**`Final Response:`** It is important to review the plan's evidence of coverage to determine if scuba diving is covered under the health plan. Additionally, discussing this with your healthcare provider and reviewing the list of excluded services and prescriptions will help clarify whether scuba diving is covered. If scuba diving is not covered under the plan, it is advisable to explore alternative coverage options or discuss payment options with your healthcare provider."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "page_label: 90\n",
      "file_path: c:\\Dev\\azure-ai-search-python-playground\\data\\pdf\\Northwind_Health_Plus_Benefits_Details.pdf\n",
      "\n",
      "benefits for mental health and \n",
      "substance abuse services as it does for medical and surgical benefits. This includes covering \n",
      "services that are medically necessary, suc h as inpatient and outpatient services, medication \n",
      "management, and psychological and psychosocial therapies.  \n",
      "It is important to note that the plan may not provide coverage or impose any limits or \n",
      "exclusions that are not in compliance with applicable laws a nd regulations. Additionally, the \n",
      "plan may not discriminate against individuals based on their medical condition or health \n",
      "status. Individuals who feel they have been discriminated against should contact the \n",
      "Department of Labor, who can investigate the iss ue. \n",
      "Finally, it is important to note that the plan may not provide coverage or impose any limits \n",
      "or exclusions that are not in compliance with applicable laws and regulations. Additionally, \n",
      "the plan may not discriminate against individuals based on their m edical condition or \n",
      "health status. Individuals who feel they have been discriminated against should contact the \n",
      "Department of Labor, who can investigate the issue.\n",
      "page_label: 75\n",
      "file_path: c:\\Dev\\azure-ai-search-python-playground\\data\\pdf\\Northwind_Health_Plus_Benefits_Details.pdf\n",
      "\n",
      "Health Plus does not cover preventive care services provided \n",
      "by a non -network  provider.  \n",
      "Tips for Avoiding Exclusions  \n",
      "When considering a medical service or treatment, it is important to review the plan's \n",
      "evidence of coverage to ensure that the service or treatment is covered under the plan. You \n",
      "should also discuss the service or treatment with your doctor to ensure that i t is medically \n",
      "necessary. Additionally, you should review the list of excluded services and prescriptions to \n",
      "ensure that you are not seeking treatment for an excluded service or prescription.  \n",
      "If you are considering a medical service or treatment that is n ot covered under the plan, you \n",
      "should discuss payment options with your doctor or healthcare provider. Additionally, you \n",
      "may need to consider other payment sources, such as private insurance, flexible spending \n",
      "accounts, or state or federal programs.\n",
      "page_label: 95\n",
      "file_path: c:\\Dev\\azure-ai-search-python-playground\\data\\pdf\\Northwind_Standard_Benefits_Details.pdf\n",
      "\n",
      "When you enroll in the Northwind Standard plan, you may be eligible to continue coverage \n",
      "under other health plans. This could include coverage from your spouse’s or a parent’s \n",
      "employer, or from a government -sponsored program such as Medica re or Medicaid.  \n",
      "If you and your spouse have coverage under different plans, you must determine which plan \n",
      "is the primary plan and which is secondary. This is important, as the primary plan will pay \n",
      "first; the secondary plan will pay what the primary plan does not.  \n",
      "If you have other coverage, you must notify Northwind Health of any changes in your \n",
      "coverage status or any changes in any of your other coverage. If you fail to do so, you may be \n",
      "responsible for any charges that Northwind Health would have paid i f you had notified \n",
      "them of the other coverage.  \n",
      "In certain circumstances, you may be able to keep your other coverage and still be eligible \n",
      "for coverage under the Northwind Standard plan. However, if the other coverage is primary, \n",
      "you will be responsible fo r any charges that would have been paid by the Northwind \n",
      "Standard plan.  \n",
      "It is also important to note that if you have coverage through a government -sponsored \n",
      "program such as Medicare or Medicaid, you may be subject to certain restrictions. For \n",
      "example, you  may be required to obtain certain services through the government -\n",
      "sponsored plan.  \n",
      "Tips for Employees  \n",
      "To ensure you get the most out of your Northwind Standard coverage, here are some tips:  \n",
      "• Make sure you understand the terms and conditions of your other coverage and any \n",
      "restrictions associated with it.  \n",
      "• Know which plan is primary and which is secondary.  \n",
      "• Notify Northwind Health of any changes in your coverage status or any changes in any of \n",
      "your other coverage.  \n",
      "• Understand any restrictions associated w ith any government -sponsored programs you \n",
      "may be enrolled in.  \n",
      "• Your Northwind Standard plan does not cover certain services, such as emergency care, \n",
      "mental health and substance abuse coverage, or out -of-network services. Be sure to explore \n",
      "alternative co verage options if you need coverage for these services.  \n",
      "• Take advantage of preventive care services and prescription drug coverage available \n",
      "through your Northwind Standard plan.  \n",
      "•\n"
     ]
    }
   ],
   "source": [
    "# Query execution\n",
    "from llama_index.core.schema import MetadataMode\n",
    "query = \"Does my health plan cover scuba diving?\"\n",
    "query_engine = index.as_query_engine(llm, similarity_top_k=3)\n",
    "response = query_engine.query(query)\n",
    "\n",
    "# Print the response\n",
    "display_response(response)\n",
    "print(\"\\n\")\n",
    "\n",
    "# Print what the LLM sees\n",
    "for node in response.source_nodes:\n",
    "    print(node.get_content(metadata_mode=MetadataMode.LLM))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare Different Query Engines\n",
    "Evaluate and compare the responses from different query engines."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from llama_index.core.query_engine import RetrieverQueryEngine\n",
    "from llama_index.core.vector_stores.types import VectorStoreQueryMode\n",
    "from llama_index.core.response.notebook_utils import display_response\n",
    "from llama_index.core import get_response_synthesizer\n",
    "import pprint\n",
    "\n",
    "# define response synthesizer\n",
    "response_synthesizer = get_response_synthesizer()\n",
    "\n",
    "# Initialize retrievers and query engines\n",
    "keyword_retriever = index.as_retriever(vector_store_query_mode=VectorStoreQueryMode.SPARSE, similarity_top_k=10)\n",
    "hybrid_retriever = index.as_retriever(vector_store_query_mode=VectorStoreQueryMode.HYBRID, similarity_top_k=10)\n",
    "semantic_hybrid_retriever = index.as_retriever(vector_store_query_mode=VectorStoreQueryMode.SEMANTIC_HYBRID, similarity_top_k=10)\n",
    "\n",
    "keyword_query_engine = RetrieverQueryEngine(retriever=keyword_retriever, response_synthesizer=response_synthesizer,)\n",
    "hybrid_query_engine = RetrieverQueryEngine(retriever=hybrid_retriever, response_synthesizer=response_synthesizer,)\n",
    "semantic_hybrid_query_engine = RetrieverQueryEngine(retriever=semantic_hybrid_retriever, response_synthesizer=response_synthesizer,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evalaute RAG with Arize AI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Your Query Engine and View Your Traces in Phoenix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "from openinference.instrumentation import using_metadata\n",
    "from phoenix.trace import using_project\n",
    "\n",
    "# Load all evaluation questions from queries.jsonl\n",
    "eval_questions = []\n",
    "with open(\"eval/queries.jsonl\", \"r\") as file:\n",
    "    for line in file:\n",
    "        # Parse each line as JSON and extract the query\n",
    "        json_line = json.loads(line.strip())\n",
    "        eval_questions.append(json_line)\n",
    "\n",
    "# List of query engines and their respective project names\n",
    "query_engines = [\n",
    "    (keyword_query_engine, \"Keyword\"),\n",
    "    (hybrid_query_engine, \"Hybrid\"),\n",
    "    (semantic_hybrid_query_engine, \"Semantic_Hybrid\"),\n",
    "]\n",
    "\n",
    "# Loop through each question and query it against each engine\n",
    "for query_data in tqdm(eval_questions):\n",
    "    query = query_data[\"query\"]\n",
    "    query_classification = query_data.get(\"query_classification\", \"undefined\")  # Default to 'undefined' if not present\n",
    "    \n",
    "    for engine, project_name in query_engines:\n",
    "        try:\n",
    "            metadata = query_classification\n",
    "            with using_project(project_name), using_metadata(metadata):\n",
    "                # Assuming the query method expects a string query and returns results\n",
    "                engine.query(query)\n",
    "        except Exception as e:\n",
    "            print(f\"Error querying {project_name} for query '{query}': {e}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export and Evaluate Your Trace Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from phoenix.evals import (\n",
    "    HallucinationEvaluator,\n",
    "    OpenAIModel,\n",
    "    QAEvaluator,\n",
    "    RelevanceEvaluator,\n",
    "    run_evals,\n",
    ")\n",
    "from phoenix.session.evaluation import get_qa_with_reference, get_retrieved_documents\n",
    "from phoenix.trace import DocumentEvaluations, SpanEvaluations\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Create queries DataFrame for each project\n",
    "keyword_queries_df = get_qa_with_reference(px.Client(), project_name=\"Keyword\")\n",
    "hybrid_queries_df = get_qa_with_reference(px.Client(), project_name=\"Hybrid\")\n",
    "semantic_hybrid_queries_df = get_qa_with_reference(px.Client(), project_name=\"Semantic_Hybrid\")\n",
    "\n",
    "# Create retrieved documents DataFrame for each project\n",
    "keyword_retrieved_documents_df = get_retrieved_documents(px.Client(), project_name=\"Keyword\")\n",
    "hybrid_retrieved_documents_df = get_retrieved_documents(px.Client(), project_name=\"Hybrid\")\n",
    "semantic_hybrid_retrieved_documents_df = get_retrieved_documents(px.Client(), project_name=\"Semantic_Hybrid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from phoenix.evals import (\n",
    "    HallucinationEvaluator,\n",
    "    OpenAIModel,\n",
    "    QAEvaluator,\n",
    "    RelevanceEvaluator,\n",
    "    run_evals,\n",
    ")\n",
    "from phoenix.session.evaluation import get_qa_with_reference, get_retrieved_documents\n",
    "from phoenix.trace import DocumentEvaluations, SpanEvaluations\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Define the evaluation model\n",
    "eval_model = OpenAIModel(\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    azure_deployment=\"gpt-4o\",  # I'm using gpt-4o for evaluation\n",
    "    model=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    api_version=\"2024-02-01\"\n",
    ")\n",
    "\n",
    "# Define evaluators\n",
    "hallucination_evaluator = HallucinationEvaluator(eval_model)\n",
    "qa_correctness_evaluator = QAEvaluator(eval_model)\n",
    "relevance_evaluator = RelevanceEvaluator(eval_model)\n",
    "\n",
    "# List of project names\n",
    "projects = [\"Keyword\", \"Hybrid\", \"Semantic_Hybrid\"]\n",
    "\n",
    "# Loop through each project and perform evaluations\n",
    "for project in projects:\n",
    "    # Create queries and retrieved documents DataFrames for the project\n",
    "    queries_df = get_qa_with_reference(px.Client(), project_name=project)\n",
    "    retrieved_documents_df = get_retrieved_documents(px.Client(), project_name=project)\n",
    "    \n",
    "    # Run evaluations\n",
    "    hallucination_eval_df, qa_correctness_eval_df = run_evals(\n",
    "        dataframe=queries_df,\n",
    "        evaluators=[hallucination_evaluator, qa_correctness_evaluator],\n",
    "        provide_explanation=True,\n",
    "    )\n",
    "    relevance_eval_df = run_evals(\n",
    "        dataframe=retrieved_documents_df,\n",
    "        evaluators=[relevance_evaluator],\n",
    "        provide_explanation=True,\n",
    "    )[0]\n",
    "    \n",
    "    # Log evaluations\n",
    "    px.Client().log_evaluations(\n",
    "        SpanEvaluations(eval_name=f\"Hallucination_{project}\", dataframe=hallucination_eval_df),\n",
    "        SpanEvaluations(eval_name=f\"QA Correctness_{project}\", dataframe=qa_correctness_eval_df),\n",
    "        DocumentEvaluations(eval_name=f\"Relevance_{project}\", dataframe=relevance_eval_df),\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Traces as Parquet file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "💾 Trace dataset saved to under ID: d7f6316d-10bc-4f64-a68a-a11ab9ed2e88\n",
      "📂 Trace dataset path: \\my_saved_traces\\trace_dataset-d7f6316d-10bc-4f64-a68a-a11ab9ed2e88.parquet\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Specify and Create the Directory for Trace Dataset\n",
    "directory = '/my_saved_traces'\n",
    "os.makedirs(directory, exist_ok=True)\n",
    "\n",
    "# Save the Trace Dataset\n",
    "trace_id = px.Client().get_trace_dataset().save(directory=directory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Traces from Directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "px.launch_app(trace=px.TraceDataset.load('f7733fda-6ad6-4427-a803-55ad2182b662', directory=\"/my_saved_traces/\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}