Chapter 7 - Building recommendation systems with LLMs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementing an LLM-powered recommendation system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('/desired/default/directory/path')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DtypeWarning: Columns (10) have mixed types. Specify dtype option on import or set low_memory=False.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adult</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>id</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>...</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>video</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 10194, 'name': 'Toy Story Collection', ...</td>\n",
       "      <td>30000000</td>\n",
       "      <td>[{'id': 16, 'name': 'Animation'}, {'id': 35, '...</td>\n",
       "      <td>http://toystory.disney.com/toy-story</td>\n",
       "      <td>862</td>\n",
       "      <td>tt0114709</td>\n",
       "      <td>en</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-10-30</td>\n",
       "      <td>373554033.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>False</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65000000</td>\n",
       "      <td>[{'id': 12, 'name': 'Adventure'}, {'id': 14, '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8844</td>\n",
       "      <td>tt0113497</td>\n",
       "      <td>en</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-15</td>\n",
       "      <td>262797249.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Roll the dice and unleash the excitement!</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>False</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 119050, 'name': 'Grumpy Old Men Collect...</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': 10749, 'name': 'Romance'}, {'id': 35, ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15602</td>\n",
       "      <td>tt0113228</td>\n",
       "      <td>en</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Still Yelling. Still Fighting. Still Ready for...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>False</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16000000</td>\n",
       "      <td>[{'id': 35, 'name': 'Comedy'}, {'id': 18, 'nam...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31357</td>\n",
       "      <td>tt0114885</td>\n",
       "      <td>en</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Cheated on, mistreated and stepped on, the wom...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>81452156.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Friends are the people who let you be yourself...</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>False</td>\n",
       "      <td>6.1</td>\n",
       "      <td>34.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 96871, 'name': 'Father of the Bride Col...</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': 35, 'name': 'Comedy'}]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11862</td>\n",
       "      <td>tt0113041</td>\n",
       "      <td>en</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Just when George Banks has recovered from his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-02-10</td>\n",
       "      <td>76578911.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Just When His World Is Back To Normal... He's ...</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>False</td>\n",
       "      <td>5.7</td>\n",
       "      <td>173.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   adult                              belongs_to_collection    budget  \\\n",
       "0  False  {'id': 10194, 'name': 'Toy Story Collection', ...  30000000   \n",
       "1  False                                                NaN  65000000   \n",
       "2  False  {'id': 119050, 'name': 'Grumpy Old Men Collect...         0   \n",
       "3  False                                                NaN  16000000   \n",
       "4  False  {'id': 96871, 'name': 'Father of the Bride Col...         0   \n",
       "\n",
       "                                              genres  \\\n",
       "0  [{'id': 16, 'name': 'Animation'}, {'id': 35, '...   \n",
       "1  [{'id': 12, 'name': 'Adventure'}, {'id': 14, '...   \n",
       "2  [{'id': 10749, 'name': 'Romance'}, {'id': 35, ...   \n",
       "3  [{'id': 35, 'name': 'Comedy'}, {'id': 18, 'nam...   \n",
       "4                     [{'id': 35, 'name': 'Comedy'}]   \n",
       "\n",
       "                               homepage     id    imdb_id original_language  \\\n",
       "0  http://toystory.disney.com/toy-story    862  tt0114709                en   \n",
       "1                                   NaN   8844  tt0113497                en   \n",
       "2                                   NaN  15602  tt0113228                en   \n",
       "3                                   NaN  31357  tt0114885                en   \n",
       "4                                   NaN  11862  tt0113041                en   \n",
       "\n",
       "                original_title  \\\n",
       "0                    Toy Story   \n",
       "1                      Jumanji   \n",
       "2             Grumpier Old Men   \n",
       "3            Waiting to Exhale   \n",
       "4  Father of the Bride Part II   \n",
       "\n",
       "                                            overview  ... release_date  \\\n",
       "0  Led by Woody, Andy's toys live happily in his ...  ...   1995-10-30   \n",
       "1  When siblings Judy and Peter discover an encha...  ...   1995-12-15   \n",
       "2  A family wedding reignites the ancient feud be...  ...   1995-12-22   \n",
       "3  Cheated on, mistreated and stepped on, the wom...  ...   1995-12-22   \n",
       "4  Just when George Banks has recovered from his ...  ...   1995-02-10   \n",
       "\n",
       "       revenue runtime                                   spoken_languages  \\\n",
       "0  373554033.0    81.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "1  262797249.0   104.0  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...   \n",
       "2          0.0   101.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "3   81452156.0   127.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "4   76578911.0   106.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "\n",
       "     status                                            tagline  \\\n",
       "0  Released                                                NaN   \n",
       "1  Released          Roll the dice and unleash the excitement!   \n",
       "2  Released  Still Yelling. Still Fighting. Still Ready for...   \n",
       "3  Released  Friends are the people who let you be yourself...   \n",
       "4  Released  Just When His World Is Back To Normal... He's ...   \n",
       "\n",
       "                         title  video vote_average vote_count  \n",
       "0                    Toy Story  False          7.7     5415.0  \n",
       "1                      Jumanji  False          6.9     2413.0  \n",
       "2             Grumpier Old Men  False          6.5       92.0  \n",
       "3            Waiting to Exhale  False          6.1       34.0  \n",
       "4  Father of the Bride Part II  False          5.7      173.0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "md = pd. read_csv('movies_metadata.csv')\n",
    "md.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adult</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>id</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>...</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>video</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 10194, 'name': 'Toy Story Collection', ...</td>\n",
       "      <td>30000000</td>\n",
       "      <td>[Animation, Comedy, Family]</td>\n",
       "      <td>http://toystory.disney.com/toy-story</td>\n",
       "      <td>862</td>\n",
       "      <td>tt0114709</td>\n",
       "      <td>en</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-10-30</td>\n",
       "      <td>373554033.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>False</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65000000</td>\n",
       "      <td>[Adventure, Fantasy, Family]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8844</td>\n",
       "      <td>tt0113497</td>\n",
       "      <td>en</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-15</td>\n",
       "      <td>262797249.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Roll the dice and unleash the excitement!</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>False</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 119050, 'name': 'Grumpy Old Men Collect...</td>\n",
       "      <td>0</td>\n",
       "      <td>[Romance, Comedy]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15602</td>\n",
       "      <td>tt0113228</td>\n",
       "      <td>en</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Still Yelling. Still Fighting. Still Ready for...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>False</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16000000</td>\n",
       "      <td>[Comedy, Drama, Romance]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31357</td>\n",
       "      <td>tt0114885</td>\n",
       "      <td>en</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Cheated on, mistreated and stepped on, the wom...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>81452156.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Friends are the people who let you be yourself...</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>False</td>\n",
       "      <td>6.1</td>\n",
       "      <td>34.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 96871, 'name': 'Father of the Bride Col...</td>\n",
       "      <td>0</td>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11862</td>\n",
       "      <td>tt0113041</td>\n",
       "      <td>en</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Just when George Banks has recovered from his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-02-10</td>\n",
       "      <td>76578911.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Just When His World Is Back To Normal... He's ...</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>False</td>\n",
       "      <td>5.7</td>\n",
       "      <td>173.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   adult                              belongs_to_collection    budget  \\\n",
       "0  False  {'id': 10194, 'name': 'Toy Story Collection', ...  30000000   \n",
       "1  False                                                NaN  65000000   \n",
       "2  False  {'id': 119050, 'name': 'Grumpy Old Men Collect...         0   \n",
       "3  False                                                NaN  16000000   \n",
       "4  False  {'id': 96871, 'name': 'Father of the Bride Col...         0   \n",
       "\n",
       "                         genres                              homepage     id  \\\n",
       "0   [Animation, Comedy, Family]  http://toystory.disney.com/toy-story    862   \n",
       "1  [Adventure, Fantasy, Family]                                   NaN   8844   \n",
       "2             [Romance, Comedy]                                   NaN  15602   \n",
       "3      [Comedy, Drama, Romance]                                   NaN  31357   \n",
       "4                      [Comedy]                                   NaN  11862   \n",
       "\n",
       "     imdb_id original_language               original_title  \\\n",
       "0  tt0114709                en                    Toy Story   \n",
       "1  tt0113497                en                      Jumanji   \n",
       "2  tt0113228                en             Grumpier Old Men   \n",
       "3  tt0114885                en            Waiting to Exhale   \n",
       "4  tt0113041                en  Father of the Bride Part II   \n",
       "\n",
       "                                            overview  ... release_date  \\\n",
       "0  Led by Woody, Andy's toys live happily in his ...  ...   1995-10-30   \n",
       "1  When siblings Judy and Peter discover an encha...  ...   1995-12-15   \n",
       "2  A family wedding reignites the ancient feud be...  ...   1995-12-22   \n",
       "3  Cheated on, mistreated and stepped on, the wom...  ...   1995-12-22   \n",
       "4  Just when George Banks has recovered from his ...  ...   1995-02-10   \n",
       "\n",
       "       revenue runtime                                   spoken_languages  \\\n",
       "0  373554033.0    81.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "1  262797249.0   104.0  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...   \n",
       "2          0.0   101.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "3   81452156.0   127.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "4   76578911.0   106.0           [{'iso_639_1': 'en', 'name': 'English'}]   \n",
       "\n",
       "     status                                            tagline  \\\n",
       "0  Released                                                NaN   \n",
       "1  Released          Roll the dice and unleash the excitement!   \n",
       "2  Released  Still Yelling. Still Fighting. Still Ready for...   \n",
       "3  Released  Friends are the people who let you be yourself...   \n",
       "4  Released  Just When His World Is Back To Normal... He's ...   \n",
       "\n",
       "                         title  video vote_average vote_count  \n",
       "0                    Toy Story  False          7.7     5415.0  \n",
       "1                      Jumanji  False          6.9     2413.0  \n",
       "2             Grumpier Old Men  False          6.5       92.0  \n",
       "3            Waiting to Exhale  False          6.1       34.0  \n",
       "4  Father of the Bride Part II  False          5.7      173.0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import ast\n",
    "\n",
    "\n",
    "# Convert string representation of dictionaries to actual dictionaries\n",
    "md['genres'] = md['genres'].apply(ast.literal_eval)\n",
    "\n",
    "# Transforming the 'genres' column\n",
    "md['genres'] = md['genres'].apply(lambda x: [genre['name'] for genre in x])\n",
    "\n",
    "\n",
    "md.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adult</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>id</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>...</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>video</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>weighted_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 10194, 'name': 'Toy Story Collection', ...</td>\n",
       "      <td>30000000</td>\n",
       "      <td>[Animation, Comedy, Family]</td>\n",
       "      <td>http://toystory.disney.com/toy-story</td>\n",
       "      <td>862</td>\n",
       "      <td>tt0114709</td>\n",
       "      <td>en</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>373554033.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>False</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>7.499658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65000000</td>\n",
       "      <td>[Adventure, Fantasy, Family]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8844</td>\n",
       "      <td>tt0113497</td>\n",
       "      <td>en</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>...</td>\n",
       "      <td>262797249.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Roll the dice and unleash the excitement!</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>False</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413.0</td>\n",
       "      <td>6.610362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 119050, 'name': 'Grumpy Old Men Collect...</td>\n",
       "      <td>0</td>\n",
       "      <td>[Romance, Comedy]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15602</td>\n",
       "      <td>tt0113228</td>\n",
       "      <td>en</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Still Yelling. Still Fighting. Still Ready for...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>False</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92.0</td>\n",
       "      <td>5.262357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16000000</td>\n",
       "      <td>[Comedy, Drama, Romance]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31357</td>\n",
       "      <td>tt0114885</td>\n",
       "      <td>en</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Cheated on, mistreated and stepped on, the wom...</td>\n",
       "      <td>...</td>\n",
       "      <td>81452156.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Friends are the people who let you be yourself...</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>False</td>\n",
       "      <td>6.1</td>\n",
       "      <td>34.0</td>\n",
       "      <td>5.079915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 96871, 'name': 'Father of the Bride Col...</td>\n",
       "      <td>0</td>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11862</td>\n",
       "      <td>tt0113041</td>\n",
       "      <td>en</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Just when George Banks has recovered from his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>76578911.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Just When His World Is Back To Normal... He's ...</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>False</td>\n",
       "      <td>5.7</td>\n",
       "      <td>173.0</td>\n",
       "      <td>5.199506</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   adult                              belongs_to_collection    budget  \\\n",
       "0  False  {'id': 10194, 'name': 'Toy Story Collection', ...  30000000   \n",
       "1  False                                                NaN  65000000   \n",
       "2  False  {'id': 119050, 'name': 'Grumpy Old Men Collect...         0   \n",
       "3  False                                                NaN  16000000   \n",
       "4  False  {'id': 96871, 'name': 'Father of the Bride Col...         0   \n",
       "\n",
       "                         genres                              homepage     id  \\\n",
       "0   [Animation, Comedy, Family]  http://toystory.disney.com/toy-story    862   \n",
       "1  [Adventure, Fantasy, Family]                                   NaN   8844   \n",
       "2             [Romance, Comedy]                                   NaN  15602   \n",
       "3      [Comedy, Drama, Romance]                                   NaN  31357   \n",
       "4                      [Comedy]                                   NaN  11862   \n",
       "\n",
       "     imdb_id original_language               original_title  \\\n",
       "0  tt0114709                en                    Toy Story   \n",
       "1  tt0113497                en                      Jumanji   \n",
       "2  tt0113228                en             Grumpier Old Men   \n",
       "3  tt0114885                en            Waiting to Exhale   \n",
       "4  tt0113041                en  Father of the Bride Part II   \n",
       "\n",
       "                                            overview  ...      revenue  \\\n",
       "0  Led by Woody, Andy's toys live happily in his ...  ...  373554033.0   \n",
       "1  When siblings Judy and Peter discover an encha...  ...  262797249.0   \n",
       "2  A family wedding reignites the ancient feud be...  ...          0.0   \n",
       "3  Cheated on, mistreated and stepped on, the wom...  ...   81452156.0   \n",
       "4  Just when George Banks has recovered from his ...  ...   76578911.0   \n",
       "\n",
       "  runtime                                   spoken_languages    status  \\\n",
       "0    81.0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "1   104.0  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...  Released   \n",
       "2   101.0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "3   127.0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "4   106.0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "\n",
       "                                             tagline  \\\n",
       "0                                                NaN   \n",
       "1          Roll the dice and unleash the excitement!   \n",
       "2  Still Yelling. Still Fighting. Still Ready for...   \n",
       "3  Friends are the people who let you be yourself...   \n",
       "4  Just When His World Is Back To Normal... He's ...   \n",
       "\n",
       "                         title  video vote_average vote_count weighted_rate  \n",
       "0                    Toy Story  False          7.7     5415.0      7.499658  \n",
       "1                      Jumanji  False          6.9     2413.0      6.610362  \n",
       "2             Grumpier Old Men  False          6.5       92.0      5.262357  \n",
       "3            Waiting to Exhale  False          6.1       34.0      5.079915  \n",
       "4  Father of the Bride Part II  False          5.7      173.0      5.199506  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate weighted rate (IMDb formula)\n",
    "def calculate_weighted_rate(vote_average, vote_count, min_vote_count=10):\n",
    "    return (vote_count / (vote_count + min_vote_count)) * vote_average + (min_vote_count / (vote_count + min_vote_count)) * 5.0\n",
    "\n",
    "# Minimum vote count to prevent skewed results\n",
    "vote_counts = md[md['vote_count'].notnull()]['vote_count'].astype('int')\n",
    "min_vote_count = vote_counts.quantile(0.95)\n",
    "\n",
    "# Create a new column 'weighted_rate'\n",
    "md['weighted_rate'] = md.apply(lambda row: calculate_weighted_rate(row['vote_average'], row['vote_count'], min_vote_count), axis=1)\n",
    "md.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = md.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genres</th>\n",
       "      <th>title</th>\n",
       "      <th>overview</th>\n",
       "      <th>weighted_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[Adventure, Action, Thriller]</td>\n",
       "      <td>GoldenEye</td>\n",
       "      <td>James Bond must unmask the mysterious head of ...</td>\n",
       "      <td>6.173464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Craig and Smokey are two guys in Los Angeles h...</td>\n",
       "      <td>6.083421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Horror, Action, Thriller, Crime]</td>\n",
       "      <td>From Dusk Till Dawn</td>\n",
       "      <td>Seth Gecko and his younger brother Richard are...</td>\n",
       "      <td>6.503176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Blue in the Face</td>\n",
       "      <td>Auggie runs a small tobacco shop in Brooklyn, ...</td>\n",
       "      <td>5.109091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Action, Adventure, Science Fiction, Family, F...</td>\n",
       "      <td>Mighty Morphin Power Rangers: The Movie</td>\n",
       "      <td>Power up with six incredible teens who out-man...</td>\n",
       "      <td>5.052129</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              genres  \\\n",
       "0                      [Adventure, Action, Thriller]   \n",
       "1                                           [Comedy]   \n",
       "2                  [Horror, Action, Thriller, Crime]   \n",
       "3                                           [Comedy]   \n",
       "4  [Action, Adventure, Science Fiction, Family, F...   \n",
       "\n",
       "                                     title  \\\n",
       "0                                GoldenEye   \n",
       "1                                   Friday   \n",
       "2                      From Dusk Till Dawn   \n",
       "3                         Blue in the Face   \n",
       "4  Mighty Morphin Power Rangers: The Movie   \n",
       "\n",
       "                                            overview  weighted_rate  \n",
       "0  James Bond must unmask the mysterious head of ...       6.173464  \n",
       "1  Craig and Smokey are two guys in Los Angeles h...       6.083421  \n",
       "2  Seth Gecko and his younger brother Richard are...       6.503176  \n",
       "3  Auggie runs a small tobacco shop in Brooklyn, ...       5.109091  \n",
       "4  Power up with six incredible teens who out-man...       5.052129  "
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_final = md[['genres', 'title', 'overview', 'weighted_rate']].reset_index(drop=True)\n",
    "md_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Title: Jurassic Park. Overview: A wealthy entrepreneur secretly creates a theme park featuring living dinosaurs drawn from prehistoric DNA. Before opening day, he invites a team of experts and his two eager grandchildren to experience the park and help calm anxious investors. However, the park is anything but amusing as the security systems go off-line and the dinosaurs escape. Genres: Adventure, Science Fiction. Rating: 7.39064935064935'"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Create a new column by combining 'title', 'overview', and 'genre'\n",
    "md_final['combined_info'] = md_final.apply(lambda row: f\"Title: {row['title']}. Overview: {row['overview']} Genres: {', '.join(row['genres'])}. Rating: {row['weighted_rate']}\", axis=1)\n",
    "md_final['combined_info'][9]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "693"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import tiktoken\n",
    "import os\n",
    "import openai\n",
    "\n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "from openai.embeddings_utils import get_embedding\n",
    "\n",
    "# embedding model parameters\n",
    "embedding_model = \"text-embedding-ada-002\"\n",
    "embedding_encoding = \"cl100k_base\"  # this the encoding for text-embedding-ada-002\n",
    "max_tokens = 8000  # the maximum for text-embedding-ada-002 is 8191\n",
    "\n",
    "encoding = tiktoken.get_encoding(embedding_encoding)\n",
    "\n",
    "# omit reviews that are too long to embed\n",
    "md_final[\"n_tokens\"] = md_final.combined_info.apply(lambda x: len(encoding.encode(x)))\n",
    "md_final = md_final[md_final.n_tokens <= max_tokens]\n",
    "len(md_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genres</th>\n",
       "      <th>title</th>\n",
       "      <th>overview</th>\n",
       "      <th>weighted_rate</th>\n",
       "      <th>combined_info</th>\n",
       "      <th>n_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[Adventure, Action, Thriller]</td>\n",
       "      <td>GoldenEye</td>\n",
       "      <td>James Bond must unmask the mysterious head of ...</td>\n",
       "      <td>6.173464</td>\n",
       "      <td>Title: GoldenEye. Overview: James Bond must un...</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Craig and Smokey are two guys in Los Angeles h...</td>\n",
       "      <td>6.083421</td>\n",
       "      <td>Title: Friday. Overview: Craig and Smokey are ...</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Horror, Action, Thriller, Crime]</td>\n",
       "      <td>From Dusk Till Dawn</td>\n",
       "      <td>Seth Gecko and his younger brother Richard are...</td>\n",
       "      <td>6.503176</td>\n",
       "      <td>Title: From Dusk Till Dawn. Overview: Seth Gec...</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Blue in the Face</td>\n",
       "      <td>Auggie runs a small tobacco shop in Brooklyn, ...</td>\n",
       "      <td>5.109091</td>\n",
       "      <td>Title: Blue in the Face. Overview: Auggie runs...</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Action, Adventure, Science Fiction, Family, F...</td>\n",
       "      <td>Mighty Morphin Power Rangers: The Movie</td>\n",
       "      <td>Power up with six incredible teens who out-man...</td>\n",
       "      <td>5.052129</td>\n",
       "      <td>Title: Mighty Morphin Power Rangers: The Movie...</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              genres  \\\n",
       "0                      [Adventure, Action, Thriller]   \n",
       "1                                           [Comedy]   \n",
       "2                  [Horror, Action, Thriller, Crime]   \n",
       "3                                           [Comedy]   \n",
       "4  [Action, Adventure, Science Fiction, Family, F...   \n",
       "\n",
       "                                     title  \\\n",
       "0                                GoldenEye   \n",
       "1                                   Friday   \n",
       "2                      From Dusk Till Dawn   \n",
       "3                         Blue in the Face   \n",
       "4  Mighty Morphin Power Rangers: The Movie   \n",
       "\n",
       "                                            overview  weighted_rate  \\\n",
       "0  James Bond must unmask the mysterious head of ...       6.173464   \n",
       "1  Craig and Smokey are two guys in Los Angeles h...       6.083421   \n",
       "2  Seth Gecko and his younger brother Richard are...       6.503176   \n",
       "3  Auggie runs a small tobacco shop in Brooklyn, ...       5.109091   \n",
       "4  Power up with six incredible teens who out-man...       5.052129   \n",
       "\n",
       "                                       combined_info  n_tokens  \n",
       "0  Title: GoldenEye. Overview: James Bond must un...        59  \n",
       "1  Title: Friday. Overview: Craig and Smokey are ...        52  \n",
       "2  Title: From Dusk Till Dawn. Overview: Seth Gec...       105  \n",
       "3  Title: Blue in the Face. Overview: Auggie runs...        87  \n",
       "4  Title: Mighty Morphin Power Rangers: The Movie...        89  "
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genres</th>\n",
       "      <th>title</th>\n",
       "      <th>overview</th>\n",
       "      <th>weighted_rate</th>\n",
       "      <th>combined_info</th>\n",
       "      <th>n_tokens</th>\n",
       "      <th>embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[Adventure, Action, Thriller]</td>\n",
       "      <td>GoldenEye</td>\n",
       "      <td>James Bond must unmask the mysterious head of ...</td>\n",
       "      <td>6.173464</td>\n",
       "      <td>Title: GoldenEye. Overview: James Bond must un...</td>\n",
       "      <td>59</td>\n",
       "      <td>[-0.023236559703946114, -0.015966948121786118,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Craig and Smokey are two guys in Los Angeles h...</td>\n",
       "      <td>6.083421</td>\n",
       "      <td>Title: Friday. Overview: Craig and Smokey are ...</td>\n",
       "      <td>52</td>\n",
       "      <td>[0.0015918031567707658, -0.010778157971799374,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Horror, Action, Thriller, Crime]</td>\n",
       "      <td>From Dusk Till Dawn</td>\n",
       "      <td>Seth Gecko and his younger brother Richard are...</td>\n",
       "      <td>6.503176</td>\n",
       "      <td>Title: From Dusk Till Dawn. Overview: Seth Gec...</td>\n",
       "      <td>105</td>\n",
       "      <td>[-0.008583318442106247, -0.004688787739723921,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Blue in the Face</td>\n",
       "      <td>Auggie runs a small tobacco shop in Brooklyn, ...</td>\n",
       "      <td>5.109091</td>\n",
       "      <td>Title: Blue in the Face. Overview: Auggie runs...</td>\n",
       "      <td>87</td>\n",
       "      <td>[-0.020235184580087662, -0.012174611911177635,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Action, Adventure, Science Fiction, Family, F...</td>\n",
       "      <td>Mighty Morphin Power Rangers: The Movie</td>\n",
       "      <td>Power up with six incredible teens who out-man...</td>\n",
       "      <td>5.052129</td>\n",
       "      <td>Title: Mighty Morphin Power Rangers: The Movie...</td>\n",
       "      <td>89</td>\n",
       "      <td>[-0.003904743120074272, -0.03925294429063797, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              genres  \\\n",
       "0                      [Adventure, Action, Thriller]   \n",
       "1                                           [Comedy]   \n",
       "2                  [Horror, Action, Thriller, Crime]   \n",
       "3                                           [Comedy]   \n",
       "4  [Action, Adventure, Science Fiction, Family, F...   \n",
       "\n",
       "                                     title  \\\n",
       "0                                GoldenEye   \n",
       "1                                   Friday   \n",
       "2                      From Dusk Till Dawn   \n",
       "3                         Blue in the Face   \n",
       "4  Mighty Morphin Power Rangers: The Movie   \n",
       "\n",
       "                                            overview  weighted_rate  \\\n",
       "0  James Bond must unmask the mysterious head of ...       6.173464   \n",
       "1  Craig and Smokey are two guys in Los Angeles h...       6.083421   \n",
       "2  Seth Gecko and his younger brother Richard are...       6.503176   \n",
       "3  Auggie runs a small tobacco shop in Brooklyn, ...       5.109091   \n",
       "4  Power up with six incredible teens who out-man...       5.052129   \n",
       "\n",
       "                                       combined_info  n_tokens  \\\n",
       "0  Title: GoldenEye. Overview: James Bond must un...        59   \n",
       "1  Title: Friday. Overview: Craig and Smokey are ...        52   \n",
       "2  Title: From Dusk Till Dawn. Overview: Seth Gec...       105   \n",
       "3  Title: Blue in the Face. Overview: Auggie runs...        87   \n",
       "4  Title: Mighty Morphin Power Rangers: The Movie...        89   \n",
       "\n",
       "                                           embedding  \n",
       "0  [-0.023236559703946114, -0.015966948121786118,...  \n",
       "1  [0.0015918031567707658, -0.010778157971799374,...  \n",
       "2  [-0.008583318442106247, -0.004688787739723921,...  \n",
       "3  [-0.020235184580087662, -0.012174611911177635,...  \n",
       "4  [-0.003904743120074272, -0.03925294429063797, ...  "
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import openai\n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "md_final[\"embedding\"] = md_final.overview.apply(lambda x: get_embedding(x, engine=embedding_model))\n",
    "md_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "md_final.rename(columns = {'embedding': 'vector'}, inplace = True)\n",
    "md_final.rename(columns = {'combined_info': 'text'}, inplace = True)\n",
    "md_final.to_pickle('movies.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start working with LLMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import LanceDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genres</th>\n",
       "      <th>title</th>\n",
       "      <th>overview</th>\n",
       "      <th>weighted_rate</th>\n",
       "      <th>text</th>\n",
       "      <th>n_tokens</th>\n",
       "      <th>vector</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[Adventure, Action, Thriller]</td>\n",
       "      <td>GoldenEye</td>\n",
       "      <td>James Bond must unmask the mysterious head of ...</td>\n",
       "      <td>6.173464</td>\n",
       "      <td>Title: GoldenEye. Overview: James Bond must un...</td>\n",
       "      <td>59</td>\n",
       "      <td>[-0.023236559703946114, -0.015966948121786118,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Craig and Smokey are two guys in Los Angeles h...</td>\n",
       "      <td>6.083421</td>\n",
       "      <td>Title: Friday. Overview: Craig and Smokey are ...</td>\n",
       "      <td>52</td>\n",
       "      <td>[0.0015918031567707658, -0.010778157971799374,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          genres      title  \\\n",
       "0  [Adventure, Action, Thriller]  GoldenEye   \n",
       "1                       [Comedy]     Friday   \n",
       "\n",
       "                                            overview  weighted_rate  \\\n",
       "0  James Bond must unmask the mysterious head of ...       6.173464   \n",
       "1  Craig and Smokey are two guys in Los Angeles h...       6.083421   \n",
       "\n",
       "                                                text  n_tokens  \\\n",
       "0  Title: GoldenEye. Overview: James Bond must un...        59   \n",
       "1  Title: Friday. Overview: Craig and Smokey are ...        52   \n",
       "\n",
       "                                              vector  \n",
       "0  [-0.023236559703946114, -0.015966948121786118,...  \n",
       "1  [0.0015918031567707658, -0.010778157971799374,...  "
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "md = pd.read_pickle('movies.pkl')\n",
    "\n",
    "md.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Title: GoldenEye. Overview: James Bond must unmask the mysterious head of the Janus Syndicate and prevent the leader from utilizing the GoldenEye weapons system to inflict devastating revenge on Britain. Genres: Adventure, Action, Thriller. Rating: 6.173464373464373'"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md['text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lancedb\n",
    "\n",
    "uri = \"data/sample-lancedb\"\n",
    "db = lancedb.connect(uri)\n",
    "table = db.create_table(\"movies\", md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import UnstructuredHTMLLoader\n",
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.vectorstores import LanceDB\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chains import RetrievalQA\n",
    "import os\n",
    "\n",
    "key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "embeddings = OpenAIEmbeddings()\n",
    "\n",
    "docsearch = LanceDB(connection = table, embedding = embeddings)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Title: Hitman: Agent 47. Overview: An assassin teams up with a woman to help her find her father and uncover the mysteries of her ancestry. Genres: Action, Crime, Thriller. Rating: 5.365800865800866', metadata={'genres': array(['Action', 'Crime', 'Thriller'], dtype=object), 'title': 'Hitman: Agent 47', 'overview': 'An assassin teams up with a woman to help her find her father and uncover the mysteries of her ancestry.', 'weighted_rate': 5.365800865800866, 'n_tokens': 52, 'vector': array([-0.00566491, -0.01658553, -0.02255735, ..., -0.01242317,\n",
       "        -0.01303058, -0.00709073], dtype=float32), '_distance': 0.42414575815200806}),\n",
       " Document(page_content='Title: A Good Day to Die Hard. Overview: Iconoclastic, take-no-prisoners cop John McClane, finds himself for the first time on foreign soil after traveling to Moscow to help his wayward son Jack - unaware that Jack is really a highly-trained CIA operative out to stop a nuclear weapons heist. With the Russian underworld in pursuit, and battling a countdown to war, the two McClanes discover that their opposing methods make them unstoppable heroes. Genres: Action, Thriller. Rating: 5.178041993422717', metadata={'genres': array(['Action', 'Thriller'], dtype=object), 'title': 'A Good Day to Die Hard', 'overview': 'Iconoclastic, take-no-prisoners cop John McClane, finds himself for the first time on foreign soil after traveling to Moscow to help his wayward son Jack - unaware that Jack is really a highly-trained CIA operative out to stop a nuclear weapons heist. With the Russian underworld in pursuit, and battling a countdown to war, the two McClanes discover that their opposing methods make them unstoppable heroes.', 'weighted_rate': 5.178041993422717, 'n_tokens': 111, 'vector': array([-0.01498934, -0.02228908, -0.01306022, ..., -0.00130485,\n",
       "        -0.00664774, -0.0015233 ], dtype=float32), '_distance': 0.42637109756469727}),\n",
       " Document(page_content='Title: Transformers. Overview: Young teenager, Sam Witwicky becomes involved in the ancient struggle between two extraterrestrial factions of transforming robots – the heroic Autobots and the evil Decepticons. Sam holds the clue to unimaginable power and the Decepticons will stop at nothing to retrieve it. Genres: Adventure, Science Fiction, Action. Rating: 6.447283923466021', metadata={'genres': array(['Adventure', 'Science Fiction', 'Action'], dtype=object), 'title': 'Transformers', 'overview': 'Young teenager, Sam Witwicky becomes involved in the ancient struggle between two extraterrestrial factions of transforming robots – the heroic Autobots and the evil Decepticons. Sam holds the clue to unimaginable power and the Decepticons will stop at nothing to retrieve it.', 'weighted_rate': 6.447283923466021, 'n_tokens': 82, 'vector': array([-0.0085541 , -0.04153363, -0.00675735, ...,  0.01361886,\n",
       "        -0.02674297, -0.00792264], dtype=float32), '_distance': 0.4277053773403168}),\n",
       " Document(page_content='Title: Ice Age: Dawn of the Dinosaurs. Overview: Times are changing for Manny the moody mammoth, Sid the motor mouthed sloth and Diego the crafty saber-toothed tiger. Life heats up for our heroes when they meet some new and none-too-friendly neighbors – the mighty dinosaurs. Genres: Animation, Comedy, Family, Adventure. Rating: 6.264471780028944', metadata={'genres': array(['Animation', 'Comedy', 'Family', 'Adventure'], dtype=object), 'title': 'Ice Age: Dawn of the Dinosaurs', 'overview': 'Times are changing for Manny the moody mammoth, Sid the motor mouthed sloth and Diego the crafty saber-toothed tiger. Life heats up for our heroes when they meet some new and none-too-friendly neighbors – the mighty dinosaurs.', 'weighted_rate': 6.264471780028944, 'n_tokens': 84, 'vector': array([-0.00151468, -0.00433621, -0.00944388, ..., -0.0069765 ,\n",
       "        -0.00752185, -0.01455156], dtype=float32), '_distance': 0.4308919906616211})]"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"I'm looking for an animated action movie. What could you suggest to me?\"\n",
    "docs = docsearch.similarity_search(query)\n",
    "docs\n",
    "# docs[0].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' I would suggest Transformers. It is an animated action movie with genres of Adventure, Science Fiction, and Action, and a rating of 6.447283923466021.'"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type=\"stuff\", retriever=docsearch.as_retriever(), return_source_documents=True)\n",
    "\n",
    "query = \"I'm looking for an animated action movie. What could you suggest to me?\"\n",
    "result = qa({\"query\": query})\n",
    "result['result']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='Title: Transformers. Overview: Young teenager, Sam Witwicky becomes involved in the ancient struggle between two extraterrestrial factions of transforming robots – the heroic Autobots and the evil Decepticons. Sam holds the clue to unimaginable power and the Decepticons will stop at nothing to retrieve it. Genres: Adventure, Science Fiction, Action. Rating: 6.447283923466021', metadata={'genres': array(['Adventure', 'Science Fiction', 'Action'], dtype=object), 'title': 'Transformers', 'overview': 'Young teenager, Sam Witwicky becomes involved in the ancient struggle between two extraterrestrial factions of transforming robots – the heroic Autobots and the evil Decepticons. Sam holds the clue to unimaginable power and the Decepticons will stop at nothing to retrieve it.', 'weighted_rate': 6.447283923466021, 'n_tokens': 82, 'vector': array([-0.0085541 , -0.04153363, -0.00675735, ...,  0.01361886,\n",
       "       -0.02674297, -0.00792264], dtype=float32), '_distance': 0.4277053773403168})"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['source_documents'][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': \"I'm looking for a movie with animals and an adventurous plot.\",\n",
       " 'result': \" Ice Age might be a good choice for you. It's an animated, comedy, family, and adventure movie about a mismatched trio of prehistoric critters who are facing an impending ice age. The rating for Ice Age is 6.892297174111213.\",\n",
       " 'source_documents': [Document(page_content='Title: Cats & Dogs 2 : The Revenge of Kitty Galore. Overview: The ongoing war between the canine and feline species is put on hold when they join forces to thwart a rogue cat spy with her own sinister plans for conquest. Genres: Comedy, Family. Rating: 4.978057553956835', metadata={'genres': array(['Comedy', 'Family'], dtype=object), 'title': 'Cats & Dogs 2 : The Revenge of Kitty Galore', 'overview': 'The ongoing war between the canine and feline species is put on hold when they join forces to thwart a rogue cat spy with her own sinister plans for conquest.', 'weighted_rate': 4.978057553956835, 'n_tokens': 66, 'vector': array([-0.00937352, -0.02231603, -0.00789624, ...,  0.00333204,\n",
       "          0.00743868, -0.02941481], dtype=float32), '_distance': 0.3443989157676697}),\n",
       "  Document(page_content=\"Title: The Curse of the Were-Rabbit. Overview: Cheese-loving eccentric Wallace and his cunning canine pal, Gromit, investigate a mystery in Nick Park's animated adventure, in which the lovable inventor and his intrepid pup run a business ridding the town of garden pests. Using only humane methods that turn their home into a halfway house for evicted vermin, the pair stumble upon a mystery involving a voracious vegetarian monster that threatens to ruin the annual veggie-growing contest. Genres: Adventure, Animation, Comedy, Family. Rating: 6.085923217550274\", metadata={'genres': array(['Adventure', 'Animation', 'Comedy', 'Family'], dtype=object), 'title': 'The Curse of the Were-Rabbit', 'overview': \"Cheese-loving eccentric Wallace and his cunning canine pal, Gromit, investigate a mystery in Nick Park's animated adventure, in which the lovable inventor and his intrepid pup run a business ridding the town of garden pests. Using only humane methods that turn their home into a halfway house for evicted vermin, the pair stumble upon a mystery involving a voracious vegetarian monster that threatens to ruin the annual veggie-growing contest.\", 'weighted_rate': 6.085923217550274, 'n_tokens': 120, 'vector': array([ 0.00895139, -0.01685125, -0.01092802, ...,  0.01110772,\n",
       "         -0.00367373, -0.02710043], dtype=float32), '_distance': 0.36180025339126587}),\n",
       "  Document(page_content='Title: Ice Age. Overview: With the impending ice age almost upon them, a mismatched trio of prehistoric critters – Manny the woolly mammoth, Diego the saber-toothed tiger and Sid the giant sloth – find an orphaned infant and decide to return it to its human parents. Along the way, the unlikely allies become friends but, when enemies attack, their quest takes on far nobler aims. Genres: Animation, Comedy, Family, Adventure. Rating: 6.892297174111213', metadata={'genres': array(['Animation', 'Comedy', 'Family', 'Adventure'], dtype=object), 'title': 'Ice Age', 'overview': 'With the impending ice age almost upon them, a mismatched trio of prehistoric critters – Manny the woolly mammoth, Diego the saber-toothed tiger and Sid the giant sloth – find an orphaned infant and decide to return it to its human parents. Along the way, the unlikely allies become friends but, when enemies attack, their quest takes on far nobler aims.', 'weighted_rate': 6.892297174111213, 'n_tokens': 107, 'vector': array([-0.0021034 , -0.02123785, -0.01054541, ...,  0.00296882,\n",
       "         -0.00765846, -0.02345653], dtype=float32), '_distance': 0.3627752959728241}),\n",
       "  Document(page_content=\"Title: Marley & Me. Overview: A newly married couple who, in the process of starting a family, learn many of life's important lessons from their trouble-loving retriever, Marley. Packed with plenty of laughs to lighten the load, the film explores the highs and lows of marriage, maturity and confronting one's own mortality, as seen through the lens of family life with a dog. Genres: Comedy, Family. Rating: 6.44841182913472\", metadata={'genres': array(['Comedy', 'Family'], dtype=object), 'title': 'Marley & Me', 'overview': \"A newly married couple who, in the process of starting a family, learn many of life's important lessons from their trouble-loving retriever, Marley. Packed with plenty of laughs to lighten the load, the film explores the highs and lows of marriage, maturity and confronting one's own mortality, as seen through the lens of family life with a dog.\", 'weighted_rate': 6.44841182913472, 'n_tokens': 98, 'vector': array([-0.00143638, -0.02463391, -0.01529539, ..., -0.00575853,\n",
       "         -0.01565957, -0.02555735], dtype=float32), '_distance': 0.3638021945953369})]}"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filtered = md[md['genres'].apply(lambda x: 'Comedy' in x)]\n",
    "qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type=\"stuff\", \n",
    "    retriever=docsearch.as_retriever(search_kwargs={'data': df_filtered}), return_source_documents=True)\n",
    "\n",
    "query = \"I'm looking for a movie with animals and an adventurous plot.\"\n",
    "result = qa({\"query\": query})\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': \"I'm looking for a movie with animals and an adventurous plot.\",\n",
       " 'result': ' Wallace and Gromit: The Curse of the Were-Rabbit, Ice Age, and Marley & Me all feature animals in adventurous plots.',\n",
       " 'source_documents': [Document(page_content='The ongoing war between the canine and feline species is put on hold when they join forces to thwart a rogue cat spy with her own sinister plans for conquest.', metadata={'adult': 'False', 'genres': array(['Comedy', 'Family'], dtype=object), 'title': 'Cats & Dogs 2 : The Revenge of Kitty Galore', 'weighted_rate': 4.978057553956835, 'n_tokens': 32, 'vector': array([-0.00937352, -0.02231603, -0.00789624, ...,  0.00333204,\n",
       "          0.00743868, -0.02941481], dtype=float32), '_distance': 0.3447684347629547}),\n",
       "  Document(page_content=\"Cheese-loving eccentric Wallace and his cunning canine pal, Gromit, investigate a mystery in Nick Park's animated adventure, in which the lovable inventor and his intrepid pup run a business ridding the town of garden pests. Using only humane methods that turn their home into a halfway house for evicted vermin, the pair stumble upon a mystery involving a voracious vegetarian monster that threatens to ruin the annual veggie-growing contest.\", metadata={'adult': 'False', 'genres': array(['Adventure', 'Animation', 'Comedy', 'Family'], dtype=object), 'title': 'The Curse of the Were-Rabbit', 'weighted_rate': 6.085923217550274, 'n_tokens': 88, 'vector': array([ 0.00895139, -0.01685125, -0.01092802, ...,  0.01110772,\n",
       "         -0.00367373, -0.02710043], dtype=float32), '_distance': 0.36215221881866455}),\n",
       "  Document(page_content='With the impending ice age almost upon them, a mismatched trio of prehistoric critters – Manny the woolly mammoth, Diego the saber-toothed tiger and Sid the giant sloth – find an orphaned infant and decide to return it to its human parents. Along the way, the unlikely allies become friends but, when enemies attack, their quest takes on far nobler aims.', metadata={'adult': 'False', 'genres': array(['Animation', 'Comedy', 'Family', 'Adventure'], dtype=object), 'title': 'Ice Age', 'weighted_rate': 6.892297174111213, 'n_tokens': 79, 'vector': array([-0.0021034 , -0.02123785, -0.01054541, ...,  0.00296882,\n",
       "         -0.00765846, -0.02345653], dtype=float32), '_distance': 0.3630030155181885}),\n",
       "  Document(page_content=\"A newly married couple who, in the process of starting a family, learn many of life's important lessons from their trouble-loving retriever, Marley. Packed with plenty of laughs to lighten the load, the film explores the highs and lows of marriage, maturity and confronting one's own mortality, as seen through the lens of family life with a dog.\", metadata={'adult': 'False', 'genres': array(['Comedy', 'Family'], dtype=object), 'title': 'Marley & Me', 'weighted_rate': 6.44841182913472, 'n_tokens': 72, 'vector': array([-0.00143638, -0.02463391, -0.01529539, ..., -0.00575853,\n",
       "         -0.01565957, -0.02555735], dtype=float32), '_distance': 0.36415261030197144})]}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type=\"stuff\", \n",
    "    retriever=docsearch.as_retriever(search_kwargs={'filter': {'adult':'False'}}), return_source_documents=True)\n",
    "\n",
    "query = \"I'm looking for a movie with animals and an adventurous plot.\"\n",
    "result = qa({\"query\": query})\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Here are some action movies that you might enjoy:\\n\\n1. [REC]² - The action continues from [REC], with a medical officer and a SWAT team sent into a sealed-off apartment to control the situation. It is a thriller/horror movie.\\n\\n2. The Boondock Saints - Twin brothers Conner and Murphy take swift retribution into their own hands to rid Boston of criminals. It is an action/thriller/crime movie.\\n\\n3. The Gamers - Four clueless players are sent on a quest to rescue a princess and must navigate dangerous forests, ancient ruins, and more. It is an action/comedy/thriller/foreign movie.\\n\\n4. Atlas Shrugged Part III: Who is John Galt? - In a collapsing economy, one man has the answer while others try to control or save him. It is a drama/science fiction/mystery movie.\\n\\nPlease note that these recommendations are based on the genre \"action\" and may vary in terms of availability and personal preferences.'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m\n",
      "Invoking: `movies` with `{'title': 'Kung Fu Panda'}`\n",
      "\n",
      "\n",
      "\u001b[0m\u001b[36;1m\u001b[1;3m[Document(page_content='When the Valley of Peace is threatened, lazy Po the panda discovers his destiny as the \"chosen one\" and trains to become a kung fu hero, but transforming the unsleek slacker into a brave warrior won\\'t be easy. It\\'s up to Master Shifu and the Furious Five -- Tigress, Crane, Mantis, Viper and Monkey -- to give it a try.', metadata={'adult': 'False', 'genres': array(['Adventure', 'Animation', 'Family', 'Comedy'], dtype=object), 'title': 'Kung Fu Panda', 'weighted_rate': 6.675006821282402, 'n_tokens': 79, 'vector': array([-0.00345203, -0.03163854,  0.00223724, ...,  0.00216131,\n",
      "        0.00014942, -0.01268781], dtype=float32), '_distance': 0.20448510348796844}), Document(page_content='Po is now living his dream as The Dragon Warrior, protecting the Valley of Peace alongside his friends and fellow kung fu masters, The Furious Five - Tigress, Crane, Mantis, Viper and Monkey. But Po’s new life of awesomeness is threatened by the emergence of a formidable villain, who plans to use a secret, unstoppable weapon to conquer China and destroy kung fu. It is up to Po and The Furious Five to journey across China to face this threat and vanquish it. But how can Po stop a weapon that can stop kung fu? He must look to his past and uncover the secrets of his mysterious origins; only then will he be able to unlock the strength he needs to succeed.', metadata={'adult': 'False', 'genres': array(['Animation', 'Family'], dtype=object), 'title': 'Kung Fu Panda 2', 'weighted_rate': 6.387240356083087, 'n_tokens': 148, 'vector': array([ 0.00179355, -0.02989138, -0.00742091, ..., -0.00236976,\n",
      "       -0.00606398, -0.00903104], dtype=float32), '_distance': 0.24536560475826263}), Document(page_content='Asian Hawk (Jackie Chan) leads a mercenary team to recover several lost artifacts from the Old Summer Palace, the bronze heads of the 12 Chinese Zodiac animals which were sacked by the French and British armies from the imperial Summer Palace in Beijing in 1860. Assisted by a Chinese student & a Parisian lady, Hawk stops at nothing to accomplish the mission.', metadata={'adult': 'False', 'genres': array(['Action', 'Adventure'], dtype=object), 'title': 'Chinese Zodiac', 'weighted_rate': 5.380000000000001, 'n_tokens': 75, 'vector': array([-0.00208469, -0.00418268, -0.00532402, ..., -0.00832544,\n",
      "        0.00562683, -0.01166626], dtype=float32), '_distance': 0.34648555517196655}), Document(page_content='Into the world of the Emperor Penguins, who find their soul mates through song, a penguin is born who cannot sing. But he can tap dance something fierce!', metadata={'adult': 'False', 'genres': array(['Animation', 'Comedy'], dtype=object), 'title': 'Happy Feet', 'weighted_rate': 5.69344262295082, 'n_tokens': 33, 'vector': array([-8.04522933e-05, -3.50862592e-02,  1.42071005e-02, ...,\n",
      "       -2.17840052e-03,  2.18484085e-03, -3.50089744e-02], dtype=float32), '_distance': 0.34831809997558594})]\u001b[0m\u001b[32;1m\u001b[1;3mBased on your liking for Kung Fu Panda 1 and 2, I would suggest the following similar movies:\n",
      "\n",
      "1. Chinese Zodiac: This action-adventure film follows Asian Hawk, played by Jackie Chan, as he leads a mercenary team to recover lost artifacts. The movie combines action and adventure elements similar to Kung Fu Panda.\n",
      "\n",
      "2. Happy Feet: This animated comedy film tells the story of a penguin who can't sing but can tap dance. While it may not have the martial arts theme, it shares the comedic and family-friendly elements found in Kung Fu Panda.\n",
      "\n",
      "These movies offer a mix of adventure, animation, and comedy, which are similar to the themes present in Kung Fu Panda.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'I liked a lot kung fu panda 1 and 2. Could you suggest me some similar movies?',\n",
       " 'chat_history': [HumanMessage(content='I liked a lot kung fu panda 1 and 2. Could you suggest me some similar movies?', additional_kwargs={}, example=False),\n",
       "  AIMessage(content='', additional_kwargs={'function_call': {'name': 'movies', 'arguments': '{\\n  \"title\": \"Kung Fu Panda\"\\n}'}}, example=False),\n",
       "  FunctionMessage(content='[Document(page_content=\\'When the Valley of Peace is threatened, lazy Po the panda discovers his destiny as the \"chosen one\" and trains to become a kung fu hero, but transforming the unsleek slacker into a brave warrior won\\\\\\'t be easy. It\\\\\\'s up to Master Shifu and the Furious Five -- Tigress, Crane, Mantis, Viper and Monkey -- to give it a try.\\', metadata={\\'adult\\': \\'False\\', \\'genres\\': array([\\'Adventure\\', \\'Animation\\', \\'Family\\', \\'Comedy\\'], dtype=object), \\'title\\': \\'Kung Fu Panda\\', \\'weighted_rate\\': 6.675006821282402, \\'n_tokens\\': 79, \\'vector\\': array([-0.00345203, -0.03163854,  0.00223724, ...,  0.00216131,\\n        0.00014942, -0.01268781], dtype=float32), \\'_distance\\': 0.20448510348796844}), Document(page_content=\\'Po is now living his dream as The Dragon Warrior, protecting the Valley of Peace alongside his friends and fellow kung fu masters, The Furious Five - Tigress, Crane, Mantis, Viper and Monkey. But Po’s new life of awesomeness is threatened by the emergence of a formidable villain, who plans to use a secret, unstoppable weapon to conquer China and destroy kung fu. It is up to Po and The Furious Five to journey across China to face this threat and vanquish it. But how can Po stop a weapon that can stop kung fu? He must look to his past and uncover the secrets of his mysterious origins; only then will he be able to unlock the strength he needs to succeed.\\', metadata={\\'adult\\': \\'False\\', \\'genres\\': array([\\'Animation\\', \\'Family\\'], dtype=object), \\'title\\': \\'Kung Fu Panda 2\\', \\'weighted_rate\\': 6.387240356083087, \\'n_tokens\\': 148, \\'vector\\': array([ 0.00179355, -0.02989138, -0.00742091, ..., -0.00236976,\\n       -0.00606398, -0.00903104], dtype=float32), \\'_distance\\': 0.24536560475826263}), Document(page_content=\\'Asian Hawk (Jackie Chan) leads a mercenary team to recover several lost artifacts from the Old Summer Palace, the bronze heads of the 12 Chinese Zodiac animals which were sacked by the French and British armies from the imperial Summer Palace in Beijing in 1860. Assisted by a Chinese student & a Parisian lady, Hawk stops at nothing to accomplish the mission.\\', metadata={\\'adult\\': \\'False\\', \\'genres\\': array([\\'Action\\', \\'Adventure\\'], dtype=object), \\'title\\': \\'Chinese Zodiac\\', \\'weighted_rate\\': 5.380000000000001, \\'n_tokens\\': 75, \\'vector\\': array([-0.00208469, -0.00418268, -0.00532402, ..., -0.00832544,\\n        0.00562683, -0.01166626], dtype=float32), \\'_distance\\': 0.34648555517196655}), Document(page_content=\\'Into the world of the Emperor Penguins, who find their soul mates through song, a penguin is born who cannot sing. But he can tap dance something fierce!\\', metadata={\\'adult\\': \\'False\\', \\'genres\\': array([\\'Animation\\', \\'Comedy\\'], dtype=object), \\'title\\': \\'Happy Feet\\', \\'weighted_rate\\': 5.69344262295082, \\'n_tokens\\': 33, \\'vector\\': array([-8.04522933e-05, -3.50862592e-02,  1.42071005e-02, ...,\\n       -2.17840052e-03,  2.18484085e-03, -3.50089744e-02], dtype=float32), \\'_distance\\': 0.34831809997558594})]', additional_kwargs={}, name='movies'),\n",
       "  AIMessage(content=\"Based on your liking for Kung Fu Panda 1 and 2, I would suggest the following similar movies:\\n\\n1. Chinese Zodiac: This action-adventure film follows Asian Hawk, played by Jackie Chan, as he leads a mercenary team to recover lost artifacts. The movie combines action and adventure elements similar to Kung Fu Panda.\\n\\n2. Happy Feet: This animated comedy film tells the story of a penguin who can't sing but can tap dance. While it may not have the martial arts theme, it shares the comedic and family-friendly elements found in Kung Fu Panda.\\n\\nThese movies offer a mix of adventure, animation, and comedy, which are similar to the themes present in Kung Fu Panda.\", additional_kwargs={}, example=False)],\n",
       " 'output': \"Based on your liking for Kung Fu Panda 1 and 2, I would suggest the following similar movies:\\n\\n1. Chinese Zodiac: This action-adventure film follows Asian Hawk, played by Jackie Chan, as he leads a mercenary team to recover lost artifacts. The movie combines action and adventure elements similar to Kung Fu Panda.\\n\\n2. Happy Feet: This animated comedy film tells the story of a penguin who can't sing but can tap dance. While it may not have the martial arts theme, it shares the comedic and family-friendly elements found in Kung Fu Panda.\\n\\nThese movies offer a mix of adventure, animation, and comedy, which are similar to the themes present in Kung Fu Panda.\",\n",
       " 'intermediate_steps': [(_FunctionsAgentAction(tool='movies', tool_input={'title': 'Kung Fu Panda'}, log=\"\\nInvoking: `movies` with `{'title': 'Kung Fu Panda'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'movies', 'arguments': '{\\n  \"title\": \"Kung Fu Panda\"\\n}'}}, example=False)]),\n",
       "   [Document(page_content='When the Valley of Peace is threatened, lazy Po the panda discovers his destiny as the \"chosen one\" and trains to become a kung fu hero, but transforming the unsleek slacker into a brave warrior won\\'t be easy. It\\'s up to Master Shifu and the Furious Five -- Tigress, Crane, Mantis, Viper and Monkey -- to give it a try.', metadata={'adult': 'False', 'genres': array(['Adventure', 'Animation', 'Family', 'Comedy'], dtype=object), 'title': 'Kung Fu Panda', 'weighted_rate': 6.675006821282402, 'n_tokens': 79, 'vector': array([-0.00345203, -0.03163854,  0.00223724, ...,  0.00216131,\n",
       "            0.00014942, -0.01268781], dtype=float32), '_distance': 0.20448510348796844}),\n",
       "    Document(page_content='Po is now living his dream as The Dragon Warrior, protecting the Valley of Peace alongside his friends and fellow kung fu masters, The Furious Five - Tigress, Crane, Mantis, Viper and Monkey. But Po’s new life of awesomeness is threatened by the emergence of a formidable villain, who plans to use a secret, unstoppable weapon to conquer China and destroy kung fu. It is up to Po and The Furious Five to journey across China to face this threat and vanquish it. But how can Po stop a weapon that can stop kung fu? He must look to his past and uncover the secrets of his mysterious origins; only then will he be able to unlock the strength he needs to succeed.', metadata={'adult': 'False', 'genres': array(['Animation', 'Family'], dtype=object), 'title': 'Kung Fu Panda 2', 'weighted_rate': 6.387240356083087, 'n_tokens': 148, 'vector': array([ 0.00179355, -0.02989138, -0.00742091, ..., -0.00236976,\n",
       "           -0.00606398, -0.00903104], dtype=float32), '_distance': 0.24536560475826263}),\n",
       "    Document(page_content='Asian Hawk (Jackie Chan) leads a mercenary team to recover several lost artifacts from the Old Summer Palace, the bronze heads of the 12 Chinese Zodiac animals which were sacked by the French and British armies from the imperial Summer Palace in Beijing in 1860. Assisted by a Chinese student & a Parisian lady, Hawk stops at nothing to accomplish the mission.', metadata={'adult': 'False', 'genres': array(['Action', 'Adventure'], dtype=object), 'title': 'Chinese Zodiac', 'weighted_rate': 5.380000000000001, 'n_tokens': 75, 'vector': array([-0.00208469, -0.00418268, -0.00532402, ..., -0.00832544,\n",
       "            0.00562683, -0.01166626], dtype=float32), '_distance': 0.34648555517196655}),\n",
       "    Document(page_content='Into the world of the Emperor Penguins, who find their soul mates through song, a penguin is born who cannot sing. But he can tap dance something fierce!', metadata={'adult': 'False', 'genres': array(['Animation', 'Comedy'], dtype=object), 'title': 'Happy Feet', 'weighted_rate': 5.69344262295082, 'n_tokens': 33, 'vector': array([-8.04522933e-05, -3.50862592e-02,  1.42071005e-02, ...,\n",
       "           -2.17840052e-03,  2.18484085e-03, -3.50089744e-02], dtype=float32), '_distance': 0.34831809997558594})])]}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.agents.openai_functions_agent.base import OpenAIFunctionsAgent\n",
    "from langchain.schema.messages import SystemMessage\n",
    "from langchain.prompts import MessagesPlaceholder\n",
    "from langchain.agents.openai_functions_agent.agent_token_buffer_memory import AgentTokenBufferMemory\n",
    "\n",
    "\n",
    "\n",
    "system_message = SystemMessage(\n",
    "        content=(\n",
    "            \"Do your best to answer the questions. \"\n",
    "            \"if there are more than one argument for the single-input tool, reason step by step and treat them as single input. \"\n",
    "            \"relevant information, only if neccessary\"\n",
    "        )\n",
    ")\n",
    "\n",
    "# This is needed for both the memory and the prompt\n",
    "memory_key = \"history\"\n",
    "\n",
    "memory = AgentTokenBufferMemory(memory_key=memory_key, llm=llm)\n",
    "\n",
    "prompt = OpenAIFunctionsAgent.create_prompt(\n",
    "        system_message=system_message,\n",
    "        extra_prompt_messages=[MessagesPlaceholder(variable_name=memory_key)]\n",
    "    )\n",
    "agent_executor = create_conversational_retrieval_agent(llm=llm, tools=tools, prompt = prompt, verbose=True)\n",
    "\n",
    "result = agent_executor({\"input\": \"I liked a lot kung fu panda 1 and 2. Could you suggest me some similar movies?\"})\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prompt engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "1. A Good Day to Die Hard: An action-packed comedy directed by John Moore, this movie follows Iconoclastic, take-no-prisoners cop John McClane as he travels to Moscow to help his wayward son Jack. With the Russian underworld in pursuit, and battling a countdown to war, the two McClanes discover that their opposing methods make them unstoppable heroes.\n",
      "2. The Hidden: An alien is on the run in America and uses the bodies of anyone in its way as a hiding place. With lots of innocent people dying in the chase, this action-packed horror movie is sure to keep you laughing.\n",
      "3. District B13: Set in the ghettos of Paris in 2010, this action-packed science fiction movie follows an undercover cop and ex-thug as they try to infiltrate a gang in order to defuse a neutron bomb. A thrilling comedy that will keep you laughing.\n"
     ]
    }
   ],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "template = \"\"\"You are a movie recommender system that help users to find movies that match their preferences. \n",
    "Use the following pieces of context to answer the question at the end. \n",
    "For each question, suggest three movies, with a short description of the plot and the reason why the user migth like it.\n",
    "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
    "\n",
    "{context}\n",
    "\n",
    "Question: {question}\n",
    "Your response:\"\"\"\n",
    "\n",
    "\n",
    "PROMPT = PromptTemplate(\n",
    "    template=template, input_variables=[\"context\", \"question\"])\n",
    "\n",
    "chain_type_kwargs = {\"prompt\": PROMPT}\n",
    "qa = RetrievalQA.from_chain_type(llm=OpenAI(), \n",
    "    chain_type=\"stuff\", \n",
    "    retriever=docsearch.as_retriever(),\n",
    "    return_source_documents=True, \n",
    "    chain_type_kwargs=chain_type_kwargs)\n",
    "\n",
    "query = \"I'm looking for a funny action movie, any suggestion?\"\n",
    "result = qa({'query':query})\n",
    "print(result['result'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are a movie recommender system that help users to find movies that match their preferences. \n",
      "Use the following pieces of context to answer the question at the end. \n",
      "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
      "\n",
      "{context}\n",
      "This is what we know about the user, and you can use this information to better tune your research:\n",
      "Age: 18\n",
      "Gender: female\n",
      "Question: {question}\n",
      "Your response:\n"
     ]
    }
   ],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "template_prefix = \"\"\"You are a movie recommender system that help users to find movies that match their preferences. \n",
    "Use the following pieces of context to answer the question at the end. \n",
    "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
    "\n",
    "{context}\"\"\"\n",
    "\n",
    "user_info = \"\"\"This is what we know about the user, and you can use this information to better tune your research:\n",
    "Age: {age}\n",
    "Gender: {gender}\"\"\"\n",
    "\n",
    "template_suffix= \"\"\"Question: {question}\n",
    "Your response:\"\"\"\n",
    "\n",
    "user_info = user_info.format(age = 18, gender = 'female')\n",
    "\n",
    "COMBINED_PROMPT = template_prefix +'\\n'+ user_info +'\\n'+ template_suffix\n",
    "print(COMBINED_PROMPT)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Sure, I can suggest some action movies for you. Here are a few examples: A Good Day to Die Hard, Goldfinger, Ong Bak 2, and The Raid 2. All of these movies have high ratings and feature thrilling action elements. I hope you find something that you enjoy!'"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PROMPT = PromptTemplate(\n",
    "    template=COMBINED_PROMPT, input_variables=[\"context\", \"question\"])\n",
    "\n",
    "chain_type_kwargs = {\"prompt\": PROMPT}\n",
    "qa = RetrievalQA.from_chain_type(llm=OpenAI(), \n",
    "    chain_type=\"stuff\", \n",
    "    retriever=docsearch.as_retriever(),\n",
    "    return_source_documents=True, \n",
    "    chain_type_kwargs=chain_type_kwargs)\n",
    "\n",
    "query = \"Can you suggest me some action movie?\"\n",
    "result = qa({'query':query})\n",
    "result['result']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Title: A Good Day to Die Hard. Overview: Iconoclastic, take-no-prisoners cop John McClane, finds himself for the first time on foreign soil after traveling to Moscow to help his wayward son Jack - unaware that Jack is really a highly-trained CIA operative out to stop a nuclear weapons heist. With the Russian underworld in pursuit, and battling a countdown to war, the two McClanes discover that their opposing methods make them unstoppable heroes. Genres: Action, Thriller. Rating: 5.178041993422717', metadata={'genres': array(['Action', 'Thriller'], dtype=object), 'title': 'A Good Day to Die Hard', 'overview': 'Iconoclastic, take-no-prisoners cop John McClane, finds himself for the first time on foreign soil after traveling to Moscow to help his wayward son Jack - unaware that Jack is really a highly-trained CIA operative out to stop a nuclear weapons heist. With the Russian underworld in pursuit, and battling a countdown to war, the two McClanes discover that their opposing methods make them unstoppable heroes.', 'weighted_rate': 5.178041993422717, 'n_tokens': 111, 'vector': array([-0.01498934, -0.02228908, -0.01306022, ..., -0.00130485,\n",
       "        -0.00664774, -0.0015233 ], dtype=float32), '_distance': 0.4059504568576813}),\n",
       " Document(page_content=\"Title: Goldfinger. Overview: Special agent 007 (Sean Connery) comes face to face with one of the most notorious villains of all time, and now he must outwit and outgun the powerful tycoon to prevent him from cashing in on a devious scheme to raid Fort Knox -- and obliterate the world's economy. Genres: Adventure, Action, Thriller. Rating: 6.536483669214732\", metadata={'genres': array(['Adventure', 'Action', 'Thriller'], dtype=object), 'title': 'Goldfinger', 'overview': \"Special agent 007 (Sean Connery) comes face to face with one of the most notorious villains of all time, and now he must outwit and outgun the powerful tycoon to prevent him from cashing in on a devious scheme to raid Fort Knox -- and obliterate the world's economy.\", 'weighted_rate': 6.536483669214732, 'n_tokens': 89, 'vector': array([-0.01302788, -0.03417065, -0.00966397, ..., -0.01510197,\n",
       "        -0.01324826, -0.00648477], dtype=float32), '_distance': 0.4087887406349182}),\n",
       " Document(page_content='Title: Ong Bak 2. Overview: Moments from death a young man is rescued by a renowned warrior. Realizing unsurpassed physical potential in the young boy he trains him into the most dangerous man alive. As he becomes a young man he goes on a lone mission of vengeance against the vicious slave traders who enslaved him as a youth and the treacherous warlord who killed his father. Genres: Adventure, Action, Thriller. Rating: 5.237966101694916', metadata={'genres': array(['Adventure', 'Action', 'Thriller'], dtype=object), 'title': 'Ong Bak 2', 'overview': 'Moments from death a young man is rescued by a renowned warrior. Realizing unsurpassed physical potential in the young boy he trains him into the most dangerous man alive. As he becomes a young man he goes on a lone mission of vengeance against the vicious slave traders who enslaved him as a youth and the treacherous warlord who killed his father.', 'weighted_rate': 5.237966101694916, 'n_tokens': 100, 'vector': array([-0.01203564, -0.02915789, -0.02014618, ..., -0.01302359,\n",
       "        -0.00216949,  0.01389806], dtype=float32), '_distance': 0.41181957721710205}),\n",
       " Document(page_content='Title: The Raid 2. Overview: After fighting his way through an apartment building populated by an army of dangerous criminals and escaping with his life, SWAT team member Rama goes undercover, joining a powerful Indonesian crime syndicate to protect his family and uncover corrupt members of his own force. Genres: Action, Crime, Thriller. Rating: 6.7086887835703', metadata={'genres': array(['Action', 'Crime', 'Thriller'], dtype=object), 'title': 'The Raid 2', 'overview': 'After fighting his way through an apartment building populated by an army of dangerous criminals and escaping with his life, SWAT team member Rama goes undercover, joining a powerful Indonesian crime syndicate to protect his family and uncover corrupt members of his own force.', 'weighted_rate': 6.7086887835703, 'n_tokens': 78, 'vector': array([ 0.00455619, -0.02637059, -0.0112055 , ..., -0.02318425,\n",
       "        -0.00706593,  0.01398521], dtype=float32), '_distance': 0.4204312264919281})]"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['source_documents']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Content based"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>movies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alice</td>\n",
       "      <td>25</td>\n",
       "      <td>F</td>\n",
       "      <td>{'Transformers: The Last Knight': 7, 'Pokémon:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bob</td>\n",
       "      <td>32</td>\n",
       "      <td>M</td>\n",
       "      <td>{'Bon Cop Bad Cop 2': 8, 'Goon: Last of the En...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  username  age gender                                             movies\n",
       "0    Alice   25      F  {'Transformers: The Last Knight': 7, 'Pokémon:...\n",
       "1      Bob   32      M  {'Bon Cop Bad Cop 2': 8, 'Goon: Last of the En..."
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data = {\n",
    "    \"username\": [\"Alice\", \"Bob\"],\n",
    "    \"age\": [25, 32],\n",
    "    \"gender\": [\"F\", \"M\"],\n",
    "    \"movies\": [\n",
    "        [(\"Transformers: The Last Knight\", 7), (\"Pokémon: Spell of the Unknown\", 5)],\n",
    "        [(\"Bon Cop Bad Cop 2\", 8), (\"Goon: Last of the Enforcers\", 9)]\n",
    "    ]\n",
    "}\n",
    "\n",
    "# Convert the \"movies\" column into dictionaries\n",
    "for i, row_movies in enumerate(data[\"movies\"]):\n",
    "    movie_dict = {}\n",
    "    for movie, rating in row_movies:\n",
    "        movie_dict[movie] = rating\n",
    "    data[\"movies\"][i] = movie_dict\n",
    "\n",
    "# Create a pandas DataFrame\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "template_prefix = \"\"\"You are a movie recommender system that help users to find movies that match their preferences. \n",
    "Use the following pieces of context to answer the question at the end. \n",
    "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
    "\n",
    "{context}\"\"\"\n",
    "\n",
    "user_info = \"\"\"This is what we know about the user, and you can use this information to better tune your research:\n",
    "Age: {age}\n",
    "Gender: {gender}\n",
    "Movies already seen alongside with rating: {movies}\"\"\"\n",
    "\n",
    "template_suffix= \"\"\"Question: {question}\n",
    "Your response:\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are a movie recommender system that help users to find movies that match their preferences. \n",
      "Use the following pieces of context to answer the question at the end. \n",
      "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
      "\n",
      "{context}\n",
      "This is what we know about the user, and you can use this information to better tune your research:\n",
      "Age: 25\n",
      "Gender: F\n",
      "Movies already seen alongside with rating: Movie: Transformers: The Last Knight, Rating: 7\n",
      "Movie: Pokémon: Spell of the Unknown, Rating: 5\n",
      "\n",
      "Question: {question}\n",
      "Your response:\n"
     ]
    }
   ],
   "source": [
    "age = df.loc[df['username']=='Alice']['age'][0]\n",
    "gender = df.loc[df['username']=='Alice']['gender'][0]\n",
    "\n",
    "movies = ''\n",
    "# Iterate over the dictionary and output movie name and rating\n",
    "for movie, rating in df['movies'][0].items():\n",
    "    output_string = f\"Movie: {movie}, Rating: {rating}\" + \"\\n\"\n",
    "    movies+=output_string\n",
    "    #print(output_string)\n",
    "user_info = user_info.format(age = age, gender = gender, movies = movies)\n",
    "\n",
    "COMBINED_PROMPT = template_prefix +'\\n'+ user_info +'\\n'+ template_suffix\n",
    "print(COMBINED_PROMPT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\" Based on your age, gender, and the movies you've already seen, I would suggest the following action movies: The Raid 2 (Action, Crime, Thriller; Rating: 6.71), Ong Bak 2 (Adventure, Action, Thriller; Rating: 5.24), Hitman: Agent 47 (Action, Crime, Thriller; Rating: 5.37), and Kingsman: The Secret Service (Crime, Comedy, Action, Adventure; Rating: 7.43).\""
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PROMPT = PromptTemplate(\n",
    "    template=COMBINED_PROMPT, input_variables=[\"context\", \"question\"])\n",
    "\n",
    "chain_type_kwargs = {\"prompt\": PROMPT}\n",
    "qa = RetrievalQA.from_chain_type(llm=OpenAI(), \n",
    "    chain_type=\"stuff\", \n",
    "    retriever=docsearch.as_retriever(),\n",
    "    return_source_documents=True, \n",
    "    chain_type_kwargs=chain_type_kwargs)\n",
    "\n",
    "query = \"Can you suggest me some action movie based on my background?\"\n",
    "result = qa({'query':query})\n",
    "result['result']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Title: The Raid 2. Overview: After fighting his way through an apartment building populated by an army of dangerous criminals and escaping with his life, SWAT team member Rama goes undercover, joining a powerful Indonesian crime syndicate to protect his family and uncover corrupt members of his own force. Genres: Action, Crime, Thriller. Rating: 6.7086887835703', metadata={'genres': array(['Action', 'Crime', 'Thriller'], dtype=object), 'title': 'The Raid 2', 'overview': 'After fighting his way through an apartment building populated by an army of dangerous criminals and escaping with his life, SWAT team member Rama goes undercover, joining a powerful Indonesian crime syndicate to protect his family and uncover corrupt members of his own force.', 'weighted_rate': 6.7086887835703, 'n_tokens': 78, 'vector': array([ 0.00455619, -0.02637059, -0.0112055 , ..., -0.02318425,\n",
       "        -0.00706593,  0.01398521], dtype=float32), '_distance': 0.4141116142272949}),\n",
       " Document(page_content='Title: Ong Bak 2. Overview: Moments from death a young man is rescued by a renowned warrior. Realizing unsurpassed physical potential in the young boy he trains him into the most dangerous man alive. As he becomes a young man he goes on a lone mission of vengeance against the vicious slave traders who enslaved him as a youth and the treacherous warlord who killed his father. Genres: Adventure, Action, Thriller. Rating: 5.237966101694916', metadata={'genres': array(['Adventure', 'Action', 'Thriller'], dtype=object), 'title': 'Ong Bak 2', 'overview': 'Moments from death a young man is rescued by a renowned warrior. Realizing unsurpassed physical potential in the young boy he trains him into the most dangerous man alive. As he becomes a young man he goes on a lone mission of vengeance against the vicious slave traders who enslaved him as a youth and the treacherous warlord who killed his father.', 'weighted_rate': 5.237966101694916, 'n_tokens': 100, 'vector': array([-0.01203564, -0.02915789, -0.02014618, ..., -0.01302359,\n",
       "        -0.00216949,  0.01389806], dtype=float32), '_distance': 0.41482454538345337}),\n",
       " Document(page_content='Title: Hitman: Agent 47. Overview: An assassin teams up with a woman to help her find her father and uncover the mysteries of her ancestry. Genres: Action, Crime, Thriller. Rating: 5.365800865800866', metadata={'genres': array(['Action', 'Crime', 'Thriller'], dtype=object), 'title': 'Hitman: Agent 47', 'overview': 'An assassin teams up with a woman to help her find her father and uncover the mysteries of her ancestry.', 'weighted_rate': 5.365800865800866, 'n_tokens': 52, 'vector': array([-0.00566491, -0.01658553, -0.02255735, ..., -0.01242317,\n",
       "        -0.01303058, -0.00709073], dtype=float32), '_distance': 0.42523619532585144}),\n",
       " Document(page_content=\"Title: Kingsman: The Secret Service. Overview: The story of a super-secret spy organization that recruits an unrefined but promising street kid into the agency's ultra-competitive training program just as a global threat emerges from a twisted tech genius. Genres: Crime, Comedy, Action, Adventure. Rating: 7.4264800861141005\", metadata={'genres': array(['Crime', 'Comedy', 'Action', 'Adventure'], dtype=object), 'title': 'Kingsman: The Secret Service', 'overview': \"The story of a super-secret spy organization that recruits an unrefined but promising street kid into the agency's ultra-competitive training program just as a global threat emerges from a twisted tech genius.\", 'weighted_rate': 7.4264800861141005, 'n_tokens': 72, 'vector': array([-0.00744663, -0.02387108, -0.00821877, ..., -0.01711808,\n",
       "        -0.03188046,  0.00557843], dtype=float32), '_distance': 0.4267725646495819})]"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['source_documents']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import tiktoken\n",
    "import os\n",
    "import openai\n",
    "\n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "from openai.embeddings_utils import get_embedding"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}