azure-ai-search-video-retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -U -q twelvelabs azure-search-documents azure-identity python-dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import requests\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.identity import DefaultAzureCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from azure.search.documents.indexes.models import (\n",
    "    AzureOpenAIModelName,\n",
    "    AzureOpenAIVectorizer,\n",
    "    AzureOpenAIVectorizerParameters,\n",
    "    BinaryQuantizationCompression,\n",
    "    HnswAlgorithmConfiguration,\n",
    "    HnswParameters,\n",
    "    ScalarQuantizationCompression,\n",
    "    ScalarQuantizationParameters,\n",
    "    SearchField,\n",
    "    SearchFieldDataType,\n",
    "    SearchIndex,\n",
    "    SearchableField,\n",
    "    SimpleField,\n",
    "    VectorSearch,\n",
    "    VectorSearchAlgorithmKind,\n",
    "    VectorSearchProfile,\n",
    ")\n",
    "from azure.search.documents.models import VectorizableTextQuery, VectorizedQuery\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load environment variables from .env file\n",
    "load_dotenv()\n",
    "\n",
    "# Set up OpenAI client based on environment variables\n",
    "AZURE_OPENAI_ENDPOINT: str = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "AZURE_OPENAI_API_KEY: str = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME: str = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\")\n",
    "AZURE_OPENAI_API_VERSION: str = \"2023-05-15\"\n",
    "\n",
    "# Set up Search Service client based on environment variables\n",
    "SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "SEARCH_SERVICE_API_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "\n",
    "INDEX_NAME = \"twleve-labs-video-search\"\n",
    "\n",
    "# Authentication method flag\n",
    "use_aad_for_search = False  # Set based on your authentication method\n",
    "\n",
    "# Choose the correct credential based on your authentication method\n",
    "credential = (\n",
    "    DefaultAzureCredential()\n",
    "    if use_aad_for_search\n",
    "    else AzureKeyCredential(SEARCH_SERVICE_API_KEY)\n",
    ")\n",
    "\n",
    "# Initialize the SearchIndexClient for creating indexes\n",
    "index_client = SearchIndexClient(\n",
    "    endpoint=SEARCH_SERVICE_ENDPOINT, credential=credential\n",
    ")\n",
    "\n",
    "# Initialize the SearchClient\n",
    "search_client = SearchClient(endpoint=SEARCH_SERVICE_ENDPOINT, credential=credential, index_name=INDEX_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from twelvelabs import TwelveLabs\n",
    "import os\n",
    "\n",
    "# Retrieve the API key from environment variables\n",
    "TWELVE_LABS_API_KEY = os.getenv('TWELVE_LABS_API_KEY')\n",
    "\n",
    "# Initialize the Twelve Labs client\n",
    "twelvelabs_client = TwelveLabs(api_key=TWELVE_LABS_API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from twelvelabs import TwelveLabs\n",
    "from twelvelabs.models.embed import EmbeddingsTask\n",
    "\n",
    "\n",
    "def generate_embedding(video_url):\n",
    "    \"\"\"\n",
    "    Generate embeddings for a given video URL using the Twelve Labs API.\n",
    "\n",
    "    This function creates an embedding task for the specified video URL using\n",
    "    the Marengo-retrieval-2.6 engine. It monitors the task progress and waits\n",
    "    for completion. Once done, it retrieves the task result and extracts the\n",
    "    embeddings along with their associated metadata.\n",
    "\n",
    "    Args:\n",
    "        video_url (str): The URL of the video to generate embeddings for.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing two elements:\n",
    "            1. list: A list of dictionaries, where each dictionary contains:\n",
    "                - 'embedding': The embedding vector as a list of floats.\n",
    "                - 'start_offset_sec': The start time of the segment in seconds.\n",
    "                - 'end_offset_sec': The end time of the segment in seconds.\n",
    "                - 'embedding_scope': The scope of the embedding (e.g., 'shot', 'scene').\n",
    "            2. EmbeddingsTaskResult: The complete task result object from Twelve Labs API.\n",
    "\n",
    "    Raises:\n",
    "        Any exceptions raised by the Twelve Labs API during task creation,\n",
    "        execution, or retrieval.\n",
    "    \"\"\"\n",
    "    # Create an embedding task\n",
    "    task = twelvelabs_client.embed.task.create(\n",
    "        engine_name=\"Marengo-retrieval-2.6\", video_url=video_url\n",
    "    )\n",
    "    print(\n",
    "        f\"Created task: id={task.id} engine_name={task.engine_name} status={task.status}\"\n",
    "    )\n",
    "\n",
    "    # Define a callback function to monitor task progress\n",
    "    def on_task_update(task: EmbeddingsTask):\n",
    "        print(f\"  Status={task.status}\")\n",
    "\n",
    "    # Wait for the task to complete\n",
    "    status = task.wait_for_done(sleep_interval=2, callback=on_task_update)\n",
    "    print(f\"Embedding done: {status}\")\n",
    "\n",
    "    # Retrieve the task result\n",
    "    task_result = twelvelabs_client.embed.task.retrieve(task.id)\n",
    "\n",
    "    # Extract and return the embeddings\n",
    "    embeddings = []\n",
    "    for v in task_result.video_embeddings:\n",
    "        embeddings.append(\n",
    "            {\n",
    "                \"embedding\": v.embedding.float,\n",
    "                \"start_offset_sec\": v.start_offset_sec,\n",
    "                \"end_offset_sec\": v.end_offset_sec,\n",
    "                \"embedding_scope\": v.embedding_scope,\n",
    "            }\n",
    "        )\n",
    "\n",
    "    return embeddings, task_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "ename": "BadRequestError",
     "evalue": "Error code: 400 - {'code': 'video_resolution_too_low', 'message': 'The resolution of the video is too low. Please use a video with resolution between 360p(480x360) and 2160p(3840x2160). Current resolution is 354p.'}",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mHTTPStatusError\u001b[0m                           Traceback (most recent call last)",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\twelvelabs\\base_client.py:31\u001b[0m, in \u001b[0;36mAPIClient._request\u001b[1;34m(self, method, url, **kwargs)\u001b[0m\n\u001b[0;32m     30\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_client\u001b[38;5;241m.\u001b[39mrequest(method, url, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m---> 31\u001b[0m     \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_for_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     32\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mTimeoutException \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\httpx\\_models.py:758\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    757\u001b[0m message \u001b[38;5;241m=\u001b[39m message\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mself\u001b[39m, error_type\u001b[38;5;241m=\u001b[39merror_type)\n\u001b[1;32m--> 758\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HTTPStatusError(message, request\u001b[38;5;241m=\u001b[39mrequest, response\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n",
      "\u001b[1;31mHTTPStatusError\u001b[0m: Client error '400 Bad Request' for url 'https://api.twelvelabs.io/v1.2/embed/tasks'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mBadRequestError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[33], line 11\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;66;03m# Example usage\u001b[39;00m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m video \u001b[38;5;129;01min\u001b[39;00m video_url:\n\u001b[1;32m---> 11\u001b[0m     embeddings, task_result \u001b[38;5;241m=\u001b[39m \u001b[43mgenerate_embedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvideo\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     13\u001b[0m     \u001b[38;5;66;03m# Add the video_url to each embedding dictionary\u001b[39;00m\n\u001b[0;32m     14\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m emb \u001b[38;5;129;01min\u001b[39;00m embeddings:\n",
      "Cell \u001b[1;32mIn[4], line 31\u001b[0m, in \u001b[0;36mgenerate_embedding\u001b[1;34m(video_url)\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;124;03mGenerate embeddings for a given video URL using the Twelve Labs API.\u001b[39;00m\n\u001b[0;32m      8\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[38;5;124;03m    execution, or retrieval.\u001b[39;00m\n\u001b[0;32m     29\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     30\u001b[0m \u001b[38;5;66;03m# Create an embedding task\u001b[39;00m\n\u001b[1;32m---> 31\u001b[0m task \u001b[38;5;241m=\u001b[39m \u001b[43mtwelvelabs_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membed\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtask\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     32\u001b[0m \u001b[43m    \u001b[49m\u001b[43mengine_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mMarengo-retrieval-2.6\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvideo_url\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvideo_url\u001b[49m\n\u001b[0;32m     33\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     34\u001b[0m \u001b[38;5;28mprint\u001b[39m(\n\u001b[0;32m     35\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCreated task: id=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtask\u001b[38;5;241m.\u001b[39mid\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m engine_name=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtask\u001b[38;5;241m.\u001b[39mengine_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m status=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtask\u001b[38;5;241m.\u001b[39mstatus\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     36\u001b[0m )\n\u001b[0;32m     38\u001b[0m \u001b[38;5;66;03m# Define a callback function to monitor task progress\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\twelvelabs\\resources\\embed.py:56\u001b[0m, in \u001b[0;36mEmbedTask.create\u001b[1;34m(self, engine_name, video_file, video_url, video_start_offset_sec, video_end_offset_sec, video_clip_length, video_embedding_scopes, **kwargs)\u001b[0m\n\u001b[0;32m     53\u001b[0m     files[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdummy\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     55\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 56\u001b[0m     res \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_post\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     57\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43membed/tasks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m     58\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremove_none_values\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     59\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     60\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     61\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     62\u001b[0m     task_id \u001b[38;5;241m=\u001b[39m res[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_id\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m     63\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mretrieve(task_id)\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\twelvelabs\\base_client.py:107\u001b[0m, in \u001b[0;36mAPIClient.post\u001b[1;34m(self, url, data, json, stream, **kwargs)\u001b[0m\n\u001b[0;32m    105\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stream:\n\u001b[0;32m    106\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request_stream(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPOST\u001b[39m\u001b[38;5;124m\"\u001b[39m, url, data\u001b[38;5;241m=\u001b[39mdata, json\u001b[38;5;241m=\u001b[39mjson, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m--> 107\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPOST\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\twelvelabs\\base_client.py:35\u001b[0m, in \u001b[0;36mAPIClient._request\u001b[1;34m(self, method, url, **kwargs)\u001b[0m\n\u001b[0;32m     33\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m APITimeoutError(request\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mgetattr\u001b[39m(response, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_request\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[0;32m     34\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mHTTPStatusError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m---> 35\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_status_error(e\u001b[38;5;241m.\u001b[39mresponse)\n\u001b[0;32m     36\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exceptions\u001b[38;5;241m.\u001b[39mAPIConnectionError(\n\u001b[0;32m     38\u001b[0m         request\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mgetattr\u001b[39m(response, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_request\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m     39\u001b[0m     )\n",
      "\u001b[1;31mBadRequestError\u001b[0m: Error code: 400 - {'code': 'video_resolution_too_low', 'message': 'The resolution of the video is too low. Please use a video with resolution between 360p(480x360) and 2160p(3840x2160). Current resolution is 354p.'}"
     ]
    }
   ],
   "source": [
    "# Define the list of video URLs\n",
    "video_url = [\n",
    "    \"https://static.videezy.com/system/resources/previews/000/004/947/original/Rainy_Road_4K_Living_Background.mp4\",\n",
    "    \"https://static.videezy.com/system/resources/previews/000/048/876/original/pattayaSM03.mp4\",\n",
    "]\n",
    "\n",
    "\n",
    "# Example usage\n",
    "for video in video_url:\n",
    "    embeddings, task_result = generate_embedding(video)\n",
    "\n",
    "    # Add the video_url to each embedding dictionary\n",
    "    for emb in embeddings:\n",
    "        emb[\"video_url\"] = video\n",
    "\n",
    "    print(f\"Generated {len(embeddings)} embeddings for the video {video}\")\n",
    "    for i, emb in enumerate(embeddings):\n",
    "        print(f\"Embedding {i+1}:\")\n",
    "        print(f\"  Scope: {emb['embedding_scope']}\")\n",
    "        print(\n",
    "            f\"  Time range: {emb['start_offset_sec']} - {emb['end_offset_sec']} seconds\"\n",
    "        )\n",
    "        print(f\"  Embedding vector (first 5 values): {emb['embedding'][:5]}\")\n",
    "        print(f\"  Video URL: {emb['video_url']}\")\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}