azure-ai-search-eval-ranx.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating Search Relevance with Azure AI Search and Cohere Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install ranx\n",
    "!pip install azure-search-documents==11.6.0b6\n",
    "!pip install azure-identity\n",
    "!pip install python-dotenv\n",
    "!pip install cohere"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Import necessary libraries and load environment variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Dev\\azure-ai-search-python-playground\\venv\\Lib\\site-packages\\pydantic\\_internal\\_config.py:341: UserWarning: Valid config keys have changed in V2:\n",
      "* 'allow_population_by_field_name' has been renamed to 'populate_by_name'\n",
      "* 'smart_union' has been removed\n",
      "  warnings.warn(message, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import cohere\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from collections import defaultdict\n",
    "from dotenv import load_dotenv\n",
    "from ranx import Qrels, Run, compare\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.identity import DefaultAzureCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from azure.search.documents.indexes.models import (\n",
    "    AIStudioModelCatalogName,\n",
    "    AzureMachineLearningParameters,\n",
    "    AzureMachineLearningVectorizer,\n",
    "    HnswAlgorithmConfiguration,\n",
    "    HnswParameters,\n",
    "    SearchField,\n",
    "    SearchFieldDataType,\n",
    "    SearchIndex,\n",
    "    SearchableField,\n",
    "    SimpleField,\n",
    "    VectorEncodingFormat,\n",
    "    VectorSearch,\n",
    "    VectorSearchAlgorithmKind,\n",
    "    VectorSearchAlgorithmMetric,\n",
    "    VectorSearchProfile\n",
    ")\n",
    "from azure.search.documents.models import (\n",
    "    VectorizableTextQuery,\n",
    "    VectorizedQuery\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Cohere API and Azure AI Search Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Environment variables\n",
    "AZURE_AI_STUDIO_COHERE_EMBED_KEY = os.getenv(\"AZURE_AI_STUDIO_COHERE_EMBED_KEY\")\n",
    "AZURE_AI_STUDIO_COHERE_EMBED_ENDPOINT = os.getenv(\"AZURE_AI_STUDIO_COHERE_EMBED_ENDPOINT\")\n",
    "AZURE_SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "AZURE_SEARCH_ADMIN_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "\n",
    "# Initialize Cohere client using Azure AI Studio\n",
    "cohere_azure_client = cohere.ClientV2(\n",
    "    base_url=f\"{AZURE_AI_STUDIO_COHERE_EMBED_ENDPOINT}/v1\", \n",
    "    api_key=AZURE_AI_STUDIO_COHERE_EMBED_KEY\n",
    ")\n",
    "\n",
    "# Initialize Azure AI Search client for two indexes (int8 and float)\n",
    "credential = AzureKeyCredential(AZURE_SEARCH_ADMIN_KEY)\n",
    "int8_index_name = \"wands-cohere-int8-index\"\n",
    "float_index_name = \"wands-cohere-float32-index\"\n",
    "int8_search_client = SearchClient(endpoint=AZURE_SEARCH_SERVICE_ENDPOINT, index_name=int8_index_name, credential=credential)\n",
    "float_search_client = SearchClient(endpoint=AZURE_SEARCH_SERVICE_ENDPOINT, index_name=float_index_name, credential=credential)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Function to generate embeddings using Cohere"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type\n",
    "from cohere.errors import (\n",
    "    TooManyRequestsError, ServiceUnavailableError, GatewayTimeoutError,\n",
    "    InternalServerError, ClientClosedRequestError\n",
    ")\n",
    "\n",
    "# Add tenacity retry mechanism to handle rate limits and other transient errors\n",
    "@retry(\n",
    "    stop=stop_after_attempt(5),  # Retry up to 5 times\n",
    "    wait=wait_fixed(5),  # Wait 5 seconds between retries\n",
    "    retry=retry_if_exception_type((\n",
    "        TooManyRequestsError,  # Rate limit error (429)\n",
    "        ServiceUnavailableError,  # Service unavailable (503)\n",
    "        GatewayTimeoutError,  # Gateway timeout (504)\n",
    "        InternalServerError,  # Internal server error (500)\n",
    "        ClientClosedRequestError  # Client closed request (499)\n",
    "    ))\n",
    ")\n",
    "def generate_embeddings(texts, input_type=\"search_document\", embedding_type=\"float\"):\n",
    "    model = \"embed-english-v3.0\"\n",
    "    texts = [texts] if isinstance(texts, str) else texts  # Ensure input is a list of strings\n",
    "    response = cohere_azure_client.embed(\n",
    "        texts=texts,\n",
    "        model=model,\n",
    "        input_type=input_type,\n",
    "        embedding_types=[embedding_type],\n",
    "    )\n",
    "    return [embedding for embedding in getattr(response.embeddings, embedding_type)]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Load Product Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_name</th>\n",
       "      <th>product_class</th>\n",
       "      <th>category hierarchy</th>\n",
       "      <th>product_description</th>\n",
       "      <th>product_features</th>\n",
       "      <th>rating_count</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>review_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>product_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>solid wood platform bed</td>\n",
       "      <td>Beds</td>\n",
       "      <td>Furniture / Bedroom Furniture / Beds &amp; Headboa...</td>\n",
       "      <td>good , deep sleep can be quite difficult to ha...</td>\n",
       "      <td>overallwidth-sidetoside:64.7|dsprimaryproducts...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>all-clad 7 qt . slow cooker</td>\n",
       "      <td>Slow Cookers</td>\n",
       "      <td>Kitchen &amp; Tabletop / Small Kitchen Appliances ...</td>\n",
       "      <td>create delicious slow-cooked meals , from tend...</td>\n",
       "      <td>capacityquarts:7|producttype : slow cooker|pro...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>98.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>all-clad electrics 6.5 qt . slow cooker</td>\n",
       "      <td>Slow Cookers</td>\n",
       "      <td>Kitchen &amp; Tabletop / Small Kitchen Appliances ...</td>\n",
       "      <td>prepare home-cooked meals on any schedule with...</td>\n",
       "      <td>features : keep warm setting|capacityquarts:6....</td>\n",
       "      <td>208.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>181.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>all-clad all professional tools pizza cutter</td>\n",
       "      <td>Slicers, Peelers And Graters</td>\n",
       "      <td>Browse By Brand / All-Clad</td>\n",
       "      <td>this original stainless tool was designed to c...</td>\n",
       "      <td>overallwidth-sidetoside:3.5|warrantylength : l...</td>\n",
       "      <td>69.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>baldwin prestige alcott passage knob with roun...</td>\n",
       "      <td>Door Knobs</td>\n",
       "      <td>Home Improvement / Doors &amp; Door Hardware / Doo...</td>\n",
       "      <td>the hardware has a rich heritage of delivering...</td>\n",
       "      <td>compatibledoorthickness:1.375 '' |countryofori...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>42.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 product_name  \\\n",
       "product_id                                                      \n",
       "0                                     solid wood platform bed   \n",
       "1                                 all-clad 7 qt . slow cooker   \n",
       "2                     all-clad electrics 6.5 qt . slow cooker   \n",
       "3                all-clad all professional tools pizza cutter   \n",
       "4           baldwin prestige alcott passage knob with roun...   \n",
       "\n",
       "                           product_class  \\\n",
       "product_id                                 \n",
       "0                                   Beds   \n",
       "1                           Slow Cookers   \n",
       "2                           Slow Cookers   \n",
       "3           Slicers, Peelers And Graters   \n",
       "4                             Door Knobs   \n",
       "\n",
       "                                           category hierarchy  \\\n",
       "product_id                                                      \n",
       "0           Furniture / Bedroom Furniture / Beds & Headboa...   \n",
       "1           Kitchen & Tabletop / Small Kitchen Appliances ...   \n",
       "2           Kitchen & Tabletop / Small Kitchen Appliances ...   \n",
       "3                                  Browse By Brand / All-Clad   \n",
       "4           Home Improvement / Doors & Door Hardware / Doo...   \n",
       "\n",
       "                                          product_description  \\\n",
       "product_id                                                      \n",
       "0           good , deep sleep can be quite difficult to ha...   \n",
       "1           create delicious slow-cooked meals , from tend...   \n",
       "2           prepare home-cooked meals on any schedule with...   \n",
       "3           this original stainless tool was designed to c...   \n",
       "4           the hardware has a rich heritage of delivering...   \n",
       "\n",
       "                                             product_features rating_count  \\\n",
       "product_id                                                                   \n",
       "0           overallwidth-sidetoside:64.7|dsprimaryproducts...         15.0   \n",
       "1           capacityquarts:7|producttype : slow cooker|pro...        100.0   \n",
       "2           features : keep warm setting|capacityquarts:6....        208.0   \n",
       "3           overallwidth-sidetoside:3.5|warrantylength : l...         69.0   \n",
       "4           compatibledoorthickness:1.375 '' |countryofori...         70.0   \n",
       "\n",
       "           average_rating review_count  \n",
       "product_id                              \n",
       "0                     4.5         15.0  \n",
       "1                     2.0         98.0  \n",
       "2                     3.0        181.0  \n",
       "3                     4.5         42.0  \n",
       "4                     5.0         42.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products_df = pd.read_csv(\"eval/products/product.csv\", sep=\"\\t\", index_col=\"product_id\", keep_default_na=False)\n",
    "\n",
    "# Extract first 5000 product names and descriptions\n",
    "product_names = products_df[\"product_name\"].tolist()[:5000]\n",
    "product_descriptions = products_df[\"product_description\"].tolist()[:5000]\n",
    "products_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Step 5: Generate and Index Float and Int8 Embeddings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Int8 embeddings for product names and descriptions...\n"
     ]
    }
   ],
   "source": [
    "print(\"Generating Int8 embeddings for product names and descriptions...\")\n",
    "product_name_int8_embeddings = generate_embeddings(product_names, embedding_type=\"int8\")\n",
    "product_description_int8_embeddings = generate_embeddings(product_descriptions, embedding_type=\"int8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Float embeddings for product names and descriptions...\n"
     ]
    }
   ],
   "source": [
    "print(\"Generating Float embeddings for product names and descriptions...\")\n",
    "product_name_float_embeddings = generate_embeddings(product_names, embedding_type=\"float\")\n",
    "product_description_float_embeddings = generate_embeddings(product_descriptions, embedding_type=\"float\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Create a Search Index in Azure AI Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "name is not a known attribute of class <class 'azure.search.documents.indexes._generated.models._models_py3.AzureMachineLearningVectorizer'> and will be ignored\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating or updating Int8 embedding index with query vectorizer...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "name is not a known attribute of class <class 'azure.search.documents.indexes._generated.models._models_py3.AzureMachineLearningVectorizer'> and will be ignored\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating or updating Float32 embedding index with query vectorizer...\n"
     ]
    }
   ],
   "source": [
    "# Function to create or update the search index for vector embeddings\n",
    "def create_or_update_index(\n",
    "    client, index_name, vector_field_type, scoring_uri, authentication_key, model_name\n",
    "):\n",
    "    # Define the search index fields based on your product schema\n",
    "    fields = [\n",
    "        SimpleField(name=\"product_id\", type=SearchFieldDataType.String, key=True),\n",
    "        SearchField(\n",
    "            name=\"product_name\",\n",
    "            type=SearchFieldDataType.String,\n",
    "            searchable=True,\n",
    "            filterable=True,\n",
    "        ),\n",
    "        SearchField(\n",
    "            name=\"product_description\", type=SearchFieldDataType.String, searchable=True\n",
    "        ),\n",
    "        SearchField(\n",
    "            name=\"product_name_vector\",\n",
    "            type=vector_field_type,\n",
    "            vector_search_dimensions=1024,\n",
    "            vector_search_profile_name=\"my-vector-config\",\n",
    "        ),\n",
    "        SearchField(\n",
    "            name=\"product_description_vector\",\n",
    "            type=vector_field_type,\n",
    "            vector_search_dimensions=1024,\n",
    "            vector_search_profile_name=\"my-vector-config\",\n",
    "        ),\n",
    "    ]\n",
    "\n",
    "    # Vector search configuration with HNSW algorithm and query vectorizer\n",
    "    vector_search = VectorSearch(\n",
    "        profiles=[\n",
    "            VectorSearchProfile(\n",
    "                name=\"my-vector-config\",\n",
    "                algorithm_configuration_name=\"my-hnsw\",\n",
    "                vectorizer_name=\"my-vectorizer\",\n",
    "            )\n",
    "        ],\n",
    "        algorithms=[\n",
    "            HnswAlgorithmConfiguration(\n",
    "                name=\"my-hnsw\",\n",
    "                kind=VectorSearchAlgorithmKind.HNSW,\n",
    "                parameters=HnswParameters(metric=VectorSearchAlgorithmMetric.COSINE),\n",
    "            )\n",
    "        ],\n",
    "        vectorizers=[\n",
    "            AzureMachineLearningVectorizer(\n",
    "                name=\"my-vectorizer\",\n",
    "                vectorizer_name=\"my-vectorizer\",\n",
    "                aml_parameters=AzureMachineLearningParameters(\n",
    "                    scoring_uri=scoring_uri,\n",
    "                    authentication_key=authentication_key,\n",
    "                    model_name=model_name,\n",
    "                ),\n",
    "            )\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    index = SearchIndex(name=index_name, fields=fields, vector_search=vector_search)\n",
    "    client.create_or_update_index(index=index)\n",
    "\n",
    "\n",
    "# Example usage: Creating indexes for both Int8 and Float32 embeddings with query vectorizer\n",
    "search_index_client = SearchIndexClient(\n",
    "    endpoint=AZURE_SEARCH_SERVICE_ENDPOINT, credential=credential\n",
    ")\n",
    "\n",
    "print(\"Creating or updating Int8 embedding index with query vectorizer...\")\n",
    "create_or_update_index(\n",
    "    search_index_client,\n",
    "    index_name=int8_index_name,\n",
    "    vector_field_type=\"Collection(Edm.SByte)\",  # Int8 embedding storage format\n",
    "    scoring_uri=AZURE_AI_STUDIO_COHERE_EMBED_ENDPOINT,\n",
    "    authentication_key=AZURE_AI_STUDIO_COHERE_EMBED_KEY,\n",
    "    model_name=AIStudioModelCatalogName.COHERE_EMBED_V3_ENGLISH,\n",
    ")\n",
    "\n",
    "print(\"Creating or updating Float32 embedding index with query vectorizer...\")\n",
    "create_or_update_index(\n",
    "    search_index_client,\n",
    "    index_name=float_index_name,\n",
    "    vector_field_type=\"Collection(Edm.Single)\",  # Float32 embedding storage format\n",
    "    scoring_uri=AZURE_AI_STUDIO_COHERE_EMBED_ENDPOINT,\n",
    "    authentication_key=AZURE_AI_STUDIO_COHERE_EMBED_KEY,\n",
    "    model_name=AIStudioModelCatalogName.COHERE_EMBED_V3_ENGLISH,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Upload Product Data and Vectors to Azure AI Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploaded 5000 documents to the index 'wands-cohere-int8-index' using buffered sender.\n",
      "Uploaded 5000 documents to the index 'wands-cohere-float32-index' using buffered sender.\n"
     ]
    }
   ],
   "source": [
    "from azure.search.documents import SearchIndexingBufferedSender\n",
    "\n",
    "# Function to upload embeddings using SearchIndexingBufferedSender\n",
    "def upload_embeddings_to_index(embeddings_name, embeddings_description, search_client, index_name, batch_size=100):\n",
    "    documents = []\n",
    "    \n",
    "    # Prepare documents with embeddings\n",
    "    for i, (name_embedding, desc_embedding) in enumerate(zip(embeddings_name, embeddings_description)):\n",
    "        document = {\n",
    "            \"product_id\": str(products_df.index[i]),\n",
    "            \"product_name\": products_df[\"product_name\"][i],\n",
    "            \"product_description\": products_df[\"product_description\"][i],\n",
    "            \"product_name_vector\": name_embedding,\n",
    "            \"product_description_vector\": desc_embedding,\n",
    "        }\n",
    "        documents.append(document)\n",
    "    \n",
    "    # Initialize SearchIndexingBufferedSender for batch uploads\n",
    "    with SearchIndexingBufferedSender(\n",
    "        endpoint=AZURE_SEARCH_SERVICE_ENDPOINT,\n",
    "        index_name=index_name,\n",
    "        credential=AzureKeyCredential(AZURE_SEARCH_ADMIN_KEY),\n",
    "        auto_flush_interval=60,  # Automatically flush every 60 seconds\n",
    "        initial_batch_action_count=batch_size  # Batch size for actions\n",
    "    ) as batch_client:\n",
    "        \n",
    "        # Upload documents in batches\n",
    "        for doc_batch in [documents[i:i + batch_size] for i in range(0, len(documents), batch_size)]:\n",
    "            batch_client.upload_documents(documents=doc_batch)\n",
    "        \n",
    "        print(f\"Uploaded {len(documents)} documents to the index '{index_name}' using buffered sender.\")\n",
    "\n",
    "# Upload embeddings to respective indexes\n",
    "upload_embeddings_to_index(product_name_int8_embeddings, product_description_int8_embeddings, int8_search_client, int8_index_name)\n",
    "upload_embeddings_to_index(product_name_float_embeddings, product_description_float_embeddings, float_search_client, float_index_name)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Load Query Data and Ground Truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>query_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>label</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25434</td>\n",
       "      <td>Exact</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12088</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>42931</td>\n",
       "      <td>Exact</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2636</td>\n",
       "      <td>Exact</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>42923</td>\n",
       "      <td>Exact</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  query_id  product_id       label  score\n",
       "0   0         0       25434       Exact     10\n",
       "1   1         0       12088  Irrelevant      0\n",
       "2   2         0       42931       Exact     10\n",
       "3   3         0        2636       Exact     10\n",
       "4   4         0       42923       Exact     10"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "queries_df = pd.read_csv(\"eval/products/query.csv\", sep=\"\\t\", index_col=\"query_id\")\n",
    "labels_df = pd.read_csv(\"eval/products/label.csv\", sep=\"\\t\")\n",
    "\n",
    "# Map ground truth labels to scores\n",
    "relevancy_scores = {\"Exact\": 10, \"Partial\": 5, \"Irrelevant\": 0}\n",
    "labels_df[\"score\"] = labels_df[\"label\"].map(relevancy_scores)\n",
    "labels_df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Perform Vector Search Using Azure AI Search\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing vector search on Int8 embeddings...\n",
      "Performing vector search on Float embeddings...\n"
     ]
    }
   ],
   "source": [
    "# Function to perform vector search using the vectorizer in Azure AI Search\n",
    "def perform_vector_search(search_client, queries_df, field):\n",
    "    run_dict = defaultdict(dict)\n",
    "    \n",
    "    for index, row in queries_df.iterrows():\n",
    "        query_text = row[\"query\"]\n",
    "        \n",
    "        # Use the vectorizer already configured in the Azure AI Search index for query embedding generation\n",
    "        vector_query = VectorizableTextQuery(text=query_text, k_nearest_neighbors=3, fields=field)\n",
    "        \n",
    "        # Perform vector search using the Azure AI Search client\n",
    "        results = search_client.search(search_text=None, vector_queries=[vector_query], top=3)\n",
    "        \n",
    "        query_id = f\"{index}\"  # Ensure query_id matches what's in qrels\n",
    "        for result in results:\n",
    "            # Use the actual product_id from the search results instead of generating a 'doc_' ID\n",
    "            product_id = result['product_id']\n",
    "            score = result['@search.score']\n",
    "            \n",
    "            # Populate the run_dict using product_id and score\n",
    "            run_dict[query_id][str(product_id)] = score\n",
    "    \n",
    "    return run_dict\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Perform vector search for both Int8 and Float embeddings\n",
    "print(\"Performing vector search on Int8 embeddings...\")\n",
    "int8_name_run_dict = perform_vector_search(int8_search_client, queries_df, \"product_name_vector\")\n",
    "int8_description_run_dict = perform_vector_search(int8_search_client, queries_df, \"product_description_vector\")\n",
    "int8_combined_run_dict = perform_vector_search(int8_search_client, queries_df, \"product_name_vector, product_description_vector\")\n",
    "\n",
    "\n",
    "print(\"Performing vector search on Float embeddings...\")\n",
    "float_name_run_dict = perform_vector_search(float_search_client, queries_df, \"product_name_vector\")\n",
    "float_description_run_dict = perform_vector_search(float_search_client, queries_df, \"product_description_vector\")\n",
    "float_combined_run_dict = perform_vector_search(float_search_client, queries_df, \"product_name_vector, product_description_vector\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10: Evaluate Search Relevance Using Ranx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Comparison Results:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_names</th>\n",
       "      <th>precision@3</th>\n",
       "      <th>recall@3</th>\n",
       "      <th>mrr@3</th>\n",
       "      <th>dcg@3</th>\n",
       "      <th>ndcg@3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>int8_product_name</td>\n",
       "      <td>0.726389</td>\n",
       "      <td>0.013993</td>\n",
       "      <td>0.816667</td>\n",
       "      <td>10.336328</td>\n",
       "      <td>0.568412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>int8_product_description</td>\n",
       "      <td>0.596528</td>\n",
       "      <td>0.011205</td>\n",
       "      <td>0.704167</td>\n",
       "      <td>8.873794</td>\n",
       "      <td>0.480235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>float_product_name</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>0.013955</td>\n",
       "      <td>0.820486</td>\n",
       "      <td>10.325911</td>\n",
       "      <td>0.568340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>float_product_description</td>\n",
       "      <td>0.596528</td>\n",
       "      <td>0.011212</td>\n",
       "      <td>0.703819</td>\n",
       "      <td>8.877886</td>\n",
       "      <td>0.480424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>int8_combined</td>\n",
       "      <td>0.674306</td>\n",
       "      <td>0.013293</td>\n",
       "      <td>0.787153</td>\n",
       "      <td>9.965795</td>\n",
       "      <td>0.543094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>float_combined</td>\n",
       "      <td>0.672222</td>\n",
       "      <td>0.013170</td>\n",
       "      <td>0.785764</td>\n",
       "      <td>9.943598</td>\n",
       "      <td>0.541845</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model_names  precision@3  recall@3     mrr@3      dcg@3  \\\n",
       "0          int8_product_name     0.726389  0.013993  0.816667  10.336328   \n",
       "1   int8_product_description     0.596528  0.011205  0.704167   8.873794   \n",
       "2         float_product_name     0.725000  0.013955  0.820486  10.325911   \n",
       "3  float_product_description     0.596528  0.011212  0.703819   8.877886   \n",
       "4              int8_combined     0.674306  0.013293  0.787153   9.965795   \n",
       "5             float_combined     0.672222  0.013170  0.785764   9.943598   \n",
       "\n",
       "     ndcg@3  \n",
       "0  0.568412  \n",
       "1  0.480235  \n",
       "2  0.568340  \n",
       "3  0.480424  \n",
       "4  0.543094  \n",
       "5  0.541845  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step 10: Evaluate search relevance using ranx\n",
    "\n",
    "# Ensure query_id and product_id columns are of type string (object)\n",
    "labels_df[\"query_id\"] = labels_df[\"query_id\"].astype(str)\n",
    "labels_df[\"product_id\"] = labels_df[\"product_id\"].astype(str)\n",
    "\n",
    "# Create qrels from labels after converting dtypes\n",
    "qrels = Qrels.from_df(labels_df, q_id_col=\"query_id\", doc_id_col=\"product_id\", score_col=\"score\")\n",
    "\n",
    "# Create runs for int8, float, and combined comparisons\n",
    "int8_name_run = Run(int8_name_run_dict, name=\"int8_product_name\")\n",
    "int8_description_run = Run(int8_description_run_dict, name=\"int8_product_description\")\n",
    "float_name_run = Run(float_name_run_dict, name=\"float_product_name\")\n",
    "float_description_run = Run(float_description_run_dict, name=\"float_product_description\")\n",
    "int8_combined_run = Run(int8_combined_run_dict, name=\"int8_combined\")\n",
    "float_combined_run = Run(float_combined_run_dict, name=\"float_combined\")\n",
    "\n",
    "# Compare search relevance metrics across different models\n",
    "report = compare(\n",
    "    qrels=qrels,\n",
    "    runs=[\n",
    "        int8_name_run, \n",
    "        int8_description_run, \n",
    "        float_name_run, \n",
    "        float_description_run, \n",
    "        int8_combined_run,   \n",
    "        float_combined_run   \n",
    "    ],\n",
    "    metrics=[\"precision@3\", \"recall@3\", \"mrr@3\", \"dcg@3\", \"ndcg@3\"],\n",
    "    make_comparable=True  # Ensure that qrels and runs have matching query IDs\n",
    ")\n",
    "\n",
    "# View results using available methods for the `Report` object\n",
    "print(\"Comparison Results:\")\n",
    "\n",
    "# Convert the report to a DataFrame and display it\n",
    "results_df = report.to_dataframe()\n",
    "results_df\n",
    "\n",
    "# Optionally, export results to a CSV\n",
    "# results_df.to_csv(\"comparison_results.csv\", index=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}