Vehicle-Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vehicle Detection and Tracking Project\n",
    "\n",
    "The main steps of this project are to:\n",
    "\n",
    "1. Train a classifer to differentiate between cars and non-cars\n",
    "2. Use the classifer to detect and track cars in a video\n",
    "\n",
    "Each step will be done as follows:\n",
    "\n",
    "#### Train a classifer\n",
    "\n",
    "1. Select features to extract from the training set\n",
    "2. Prepare the data\n",
    "3. Train a classifier\n",
    "4. Test the classifer\n",
    "\n",
    "Once a classifer has been sufficiently trained and the overall accuracy is acceptable, it can be used without retraining.\n",
    "\n",
    "#### Vehicle detection\n",
    "\n",
    "1. Use a sliding windows of various sizes to search for positive matches\n",
    "2. Convert the detected matches into a heatmap to combine multiple detections \n",
    "3. Threshold the heatmap to remove false positives\n",
    "4. Draw a bounding box over the original frame of the video using the heatmap to determine location and size\n",
    "\n",
    "---\n",
    "\n",
    "### Imports\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import glob\n",
    "import itertools\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import random\n",
    "\n",
    "from scipy.ndimage.measurements import label\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from skimage.feature import hog\n",
    "\n",
    "try:\n",
    "    # scikit-learn version >= 0.18\n",
    "    from sklearn.model_selection import train_test_split\n",
    "except:\n",
    "    # scikit-learn version <= 0.17\n",
    "    from sklearn.cross_validation import train_test_split\n",
    "\n",
    "# visualizations will be shown in the notebook\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "---\n",
    "\n",
    "## Train a classifier\n",
    "\n",
    "### 0. Read in the datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_data():\n",
    "    \n",
    "    \"\"\"Read in the data to train and validate the classifier\n",
    "    \n",
    "    Returns:\n",
    "        A tuple of lists for images of vehicles and images of non-vehicles\"\"\"\n",
    "    \n",
    "    # Used for testing purposes while flushing out the pipeline\n",
    "    VEHICLES_SMALL = 'data/vehicles_smallset/*/*.jpeg'\n",
    "    NON_VEHICLES_SMALL = 'data/non-vehicles_smallset/*/*.jpeg'\n",
    "\n",
    "    # Used for training the final model in some form or another\n",
    "    VEHICLES_FULL = 'data/vehicles/*/*.png'\n",
    "    NON_VEHICLES_FULL = 'data/non-vehicles/*/*.png'\n",
    "    \n",
    "    vehicles = glob.glob(VEHICLES_FULL)\n",
    "    non_vehicles = glob.glob(NON_VEHICLES_FULL)\n",
    "\n",
    "    return (vehicles, non_vehicles)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cars, notcars = read_data()\n",
    "\n",
    "print('cars: {}, notcars: {}'.format(len(cars), len(notcars)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_collage(filenames):\n",
    "    \n",
    "    \"\"\"Create a collage of the images\n",
    "    \n",
    "    Parameters:\n",
    "        • filenames - a list of image filenames\n",
    "        \n",
    "    Returns:\n",
    "        A collage image\"\"\"\n",
    "    \n",
    "    cols = math.ceil(math.sqrt(len(filenames)))\n",
    "    rows = math.ceil(len(filenames) / cols)\n",
    "\n",
    "    w, h, d = (64, 64, 3)\n",
    "\n",
    "    collage = np.zeros((cols * w, rows * h, d), dtype='uint8')\n",
    "\n",
    "    col = row = 0\n",
    "\n",
    "    for i, filename in enumerate(filenames):\n",
    "        img = cv2.imread(filename)        \n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "    \n",
    "        x_pos = col * w\n",
    "        y_pos = row * h\n",
    "        collage[x_pos:x_pos + w, y_pos:y_pos + h, :] = img\n",
    "    \n",
    "        col += 1\n",
    "        if col >= cols:\n",
    "            col = 0\n",
    "            row += 1\n",
    "            \n",
    "    return collage\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for data_set, title in zip((cars, notcars), ('Cars', 'Non-cars')):\n",
    "    collage = create_collage(random.sample(data_set, min(len(data_set), 9)))\n",
    "    plt.figure()\n",
    "    plt.title(title)\n",
    "    plt.imshow(collage);\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Feature selection\n",
    "\n",
    "Select the features, on which to train the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_color(img, to_color_space='RGB', from_color_space='BGR'):\n",
    "\n",
    "    \"\"\"Helper function to convert an image from one color space to another.\n",
    "    The assumption is that the image was read in using OpenCV, hence the\n",
    "    'BGR' color space default.\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image to convert\n",
    "        • to_color_space - desired color space (default: 'RGB')\n",
    "        • from_color_space - input color space (default: 'BGR')\n",
    "        \n",
    "    Returns:\n",
    "        An image in the new color space or the original image if there was an error\"\"\"\n",
    "    \n",
    "    if to_color_space == from_color_space:\n",
    "        converted_img = np.copy(img)\n",
    "    else:\n",
    "        try:\n",
    "            # get the conversion identifier to use\n",
    "            conversion = getattr(cv2, 'COLOR_{}2{}'.format(from_color_space, to_color_space))\n",
    "        except AttributeError as e:\n",
    "            print(\"ERROR: {}\".format(e.args))\n",
    "            return img\n",
    "\n",
    "        # convert image to new color space (if specified)\n",
    "        converted_img = cv2.cvtColor(img, conversion)\n",
    "\n",
    "    return converted_img\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def CLAHE(img, from_color_space='BGR', tile_size=4):\n",
    "\n",
    "    ### Contrast Limited Adaptive Histogram Equalization\n",
    "    ### http://docs.opencv.org/3.1.0/d5/daf/tutorial_py_histogram_equalization.html\n",
    "\n",
    "    if len(img.shape) > 2 and img.shape[2] > 1:\n",
    "        y = convert_color(img, 'YUV', from_color_space)[:, :, 0]\n",
    "    elif len(img.shape) > 2:\n",
    "        y = img[:, :, 0]\n",
    "    else:\n",
    "        y = img\n",
    "\n",
    "    clahe = cv2.createCLAHE(tileGridSize=(tile_size, tile_size))\n",
    "    return clahe.apply(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bin_spatial(img, size=(32, 32)):\n",
    "\n",
    "    \"\"\"Extract the spatial binned color features\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image\n",
    "        • size - reduced size of image to use as features\n",
    "        \n",
    "    Returns:\n",
    "        A list of color features based on the resized image\"\"\"\n",
    "\n",
    "    # use cv2.resize().ravel() to create the feature vector\n",
    "    features = cv2.resize(img, size).ravel()\n",
    "\n",
    "    # Return the feature vector\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def color_hist(img, nbins=32, bins_range=(0, 256)):\n",
    "\n",
    "    \"\"\"Calculate a histogram for each color channel in the image and create a list of features from them.\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image\n",
    "        • nbins - number of bins in the histogram\n",
    "        • bins_range - lower and upper range of the bins (above and below are ignored)\n",
    "        \n",
    "    Returns:\n",
    "        A list of color histogram features for the image\"\"\"\n",
    "    \n",
    "    # Compute the histogram of the color channels separately\n",
    "    if len(img.shape) > 2:\n",
    "        hist_features = np.concatenate([np.histogram(img[:, :, c], bins=nbins, range=bins_range)[0] for c in range(img.shape[-1])])\n",
    "    else:\n",
    "        hist_features = np.array([np.histogram(img, bins=nbins, range=bins_range)[0]])\n",
    "        \n",
    "    # return the individual histograms, bin_centers and feature vector\n",
    "    return hist_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_hog_features(img, orient, pix_per_cell, cell_per_block, vis=False, feature_vec=True):\n",
    "    \n",
    "    \"\"\"Extract the Histogram of Oriented Gradient (HOG) features for the image.\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image\n",
    "        • orient - number of orientations for HOG features\n",
    "        • pix_per_cell - cell size over which each gradient histogram is computed\n",
    "        • cell_per_block - specifies the local area over which the histogram counts in a given cell will be normalized\n",
    "        • vis - boolean to enable a visualization of the HOG\n",
    "        • feature_vec - boolean to return the data as a feature vector\n",
    "        \n",
    "    Returns:\n",
    "        The HOG features will be returned either multidimensional or as a feature vector depending on `feature_vec`.\n",
    "        If `vis` is true, an image representation of the HOG is also returned.\"\"\"\n",
    "    \n",
    "    # Call with two outputs if vis==True\n",
    "    if vis == True:\n",
    "        features, hog_image = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),\n",
    "                                  cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=True,\n",
    "                                  visualise=vis, feature_vector=feature_vec)\n",
    "        return features, hog_image\n",
    "    # Otherwise call with one output\n",
    "    else:\n",
    "        features = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),\n",
    "                       cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=True,\n",
    "                       visualise=vis, feature_vector=feature_vec)\n",
    "        return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_features(imgs, color_space='RGB', spatial_size=(32, 32),\n",
    "                        hist_bins=32, orient=9,\n",
    "                        pix_per_cell=8, cell_per_block=2, hog_channel=0, hog_clahe=True,\n",
    "                        spatial_feat=True, hist_feat=True, hog_feat=True):\n",
    "    \n",
    "    \"\"\"Extract features from the input images based on the parameters passed in.\n",
    "    \n",
    "    Parameters:\n",
    "        • imgs - a list of input images\n",
    "        • color_space - desired color space to extract features\n",
    "        • spatial_size - size for spacial binning of color features\n",
    "        • hist_bins - number of bins for the color histogram features\n",
    "        • orient - number of orientations for HOG features\n",
    "        • pix_per_cell - cell size over which each gradient histogram is computed\n",
    "        • cell_per_block - specifies the local area over which the histogram counts in a given cell will be normalized\n",
    "        • hog_channel - image channel to apply the Histogram of Oriented Gradient (HOG)\n",
    "        • hog_clahe - boolean to use the Contrast Limited Adaptive Histogram Equalization for the HOG input\n",
    "        • spatial_feat - boolean to enable spatial binning of color features\n",
    "        • hist_feat - boolean to enable color histogram features\n",
    "        • hog_feat - boolean to enable HOG features\n",
    "        \n",
    "    Returns:\n",
    "        A list of features per image\"\"\"\n",
    "    \n",
    "    # create a list to append feature vectors to\n",
    "    features = []\n",
    "    \n",
    "    # iterate through the list of images\n",
    "    for file in imgs:\n",
    "\n",
    "        # features for this single file\n",
    "        file_features = []\n",
    "\n",
    "        # read in each one by one\n",
    "        img = cv2.imread(file)\n",
    "\n",
    "        # apply color conversion if other than 'RGB'\n",
    "        feature_img = convert_color(img, color_space)\n",
    "\n",
    "        # normalize the image\n",
    "        #feature_img = feature_img.astype(np.float32)/255\n",
    "        \n",
    "        # extract spatial binning of color features, if enabled\n",
    "        if spatial_feat:\n",
    "            spatial_features = bin_spatial(feature_img, size=spatial_size)\n",
    "            file_features.append(spatial_features)\n",
    "            \n",
    "        # extract color histogram features, if enabled\n",
    "        if hist_feat:\n",
    "            hist_features = color_hist(feature_img, nbins=hist_bins)\n",
    "            file_features.append(hist_features)\n",
    "            \n",
    "        if hog_feat:\n",
    "            if hog_clahe:\n",
    "                feature_img[:, :, 0] = CLAHE(feature_img)\n",
    "                hog_channel = (hog_channel == 'ALL') and hog_channel or 0\n",
    "            if hog_channel == 'ALL':\n",
    "                hog_features = []\n",
    "                for channel in range(feature_img.shape[2]):\n",
    "                    hog_features.append(get_hog_features(feature_img[:,:,channel],\n",
    "                                        orient, pix_per_cell, cell_per_block,\n",
    "                                        vis=False, feature_vec=True))\n",
    "                hog_features = np.ravel(hog_features)\n",
    "            else:\n",
    "                hog_features = get_hog_features(feature_img[:,:,hog_channel], orient,\n",
    "                            pix_per_cell, cell_per_block, vis=False, feature_vec=True)\n",
    "            # Append the new feature vector to the features list\n",
    "            file_features.append(hog_features)\n",
    "        features.append(np.concatenate(file_features))\n",
    "    # Return list of feature vectors\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "color_space = 'YUV' # can be RGB, HSV, LUV, HLS, YUV, YCrCb\n",
    "orient = 9  # HOG orientations\n",
    "pix_per_cell = 8 # HOG pixels per cell\n",
    "cell_per_block = 2 # HOG cells per block\n",
    "hog_channel = \"ALL\" # can be 0, 1, 2, or \"ALL\"\n",
    "hog_clahe = False # use the CLAHE for calculating the HOG\n",
    "spatial_size = (16, 16) # spatial binning dimensions\n",
    "hist_bins = 16    # number of histogram bins\n",
    "spatial_feat = True # spatial features on or off\n",
    "hist_feat = True # histogram features on or off\n",
    "hog_feat = True # HOG features on or off\n",
    "\n",
    "car_features = extract_features(cars, color_space=color_space,\n",
    "                        spatial_size=spatial_size, hist_bins=hist_bins,\n",
    "                        orient=orient, pix_per_cell=pix_per_cell,\n",
    "                        cell_per_block=cell_per_block,\n",
    "                        hog_channel=hog_channel, hog_clahe=hog_clahe, spatial_feat=spatial_feat,\n",
    "                        hist_feat=hist_feat, hog_feat=hog_feat)\n",
    "notcar_features = extract_features(notcars, color_space=color_space,\n",
    "                        spatial_size=spatial_size, hist_bins=hist_bins,\n",
    "                        orient=orient, pix_per_cell=pix_per_cell,\n",
    "                        cell_per_block=cell_per_block,\n",
    "                        hog_channel=hog_channel, hog_clahe=hog_clahe, spatial_feat=spatial_feat,\n",
    "                        hist_feat=hist_feat, hog_feat=hog_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Prepare the data\n",
    "\n",
    "Prepare the data for training the classifier. Data needs to be normalized, shuffled and split into training and testing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_data(true_features, false_features):\n",
    "    \n",
    "    \"\"\"Helper function to prepare the data. This function will normalize the inputs and create the appropriate labels.\n",
    "    \n",
    "    Parameters:\n",
    "        • true_features - features associated with a positive response\n",
    "        • false_features - features associated with a negative response\n",
    "        \n",
    "    Returns:\n",
    "        A tuple of input features (X), associated labels (y), and the scaler used\"\"\"\n",
    "    \n",
    "    # create an array stack of feature vectors\n",
    "    X = np.vstack((car_features, notcar_features)).astype(np.float64)\n",
    "    \n",
    "    # fit a per-column scaler\n",
    "    scaler = StandardScaler().fit(X)\n",
    "    \n",
    "    # apply the scaler to X\n",
    "    scaled_X = scaler.transform(X)\n",
    "    \n",
    "    # define the labels vector\n",
    "    y = np.hstack((np.ones(len(true_features)), np.zeros(len(false_features))))\n",
    "    \n",
    "    return scaled_X, y, scaler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def split_data(X, y, test_size=0.2, random_state=None):\n",
    "    \n",
    "    \"\"\"Helper function to split data into training and testing data\n",
    "    \n",
    "    Parameters:\n",
    "        • X - input features matrix\n",
    "        • y - associated labels\n",
    "        • test_size - percentage of the data to be used for testing [0.0, 1.0]\n",
    "        • random_state - number to determine the random state of the algorithm. Useful for consistent testing.\n",
    "        \n",
    "    Returns:\n",
    "        A tuple of X training, X testing, y training, y testing split appropriately.\"\"\"\n",
    "    \n",
    "    # select the random state to initialize\n",
    "    random_state = random_state is None and np.random.randint(0, 100) or random_state\n",
    "\n",
    "    # split up data into randomized training and test sets\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=random_state)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create the input feature matrix and the labels for the data\n",
    "X, y, scaler = prepare_data(car_features, notcar_features)\n",
    "\n",
    "# split the data into training and testing\n",
    "X_train, X_test, y_train, y_test = split_data(X, y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train a classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_classifier(X, y, classifier='LinearSVM'):\n",
    "    \n",
    "    \"\"\"Create and train a classifier.\n",
    "    \n",
    "    Parameters:\n",
    "        • X - input feature matrix\n",
    "        • y - labels\n",
    "        • classifer - string indicating desired classifer type\n",
    "        \n",
    "    Returns:\n",
    "        A trained classifier\"\"\"\n",
    "    \n",
    "    # select the classifier\n",
    "    if classifier == 'LinearSVM':\n",
    "        c = LinearSVC()\n",
    "    else:\n",
    "        print(\"WARNING: {} is not currently supported, defaulting to 'LinearSVM'\")\n",
    "        c = LinearSVC()\n",
    "      \n",
    "    # train the classifier\n",
    "    c.fit(X, y)\n",
    "    \n",
    "    return c\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a linear SVM\n",
    "svm = train_classifier(X_train, y_train, classifier='LinearSVM')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Test the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_classifier(c, X, y):\n",
    "    \n",
    "    \"\"\"Test the classifier on the inputs provided\n",
    "    \n",
    "    Parameter:\n",
    "        • c - trained classifier\n",
    "        • X - input feature matrix\n",
    "        • y - labels\n",
    "        \n",
    "    Returns:\n",
    "        A float representing the accuracy of the model on this test data.\"\"\"\n",
    "    \n",
    "    return c.score(X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# test the classifier\n",
    "accuracy = test_classifier(svm, X_test, y_test)\n",
    "\n",
    "print('Classifier accuracy: {:.2f}%'.format(accuracy * 100))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Save the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_classifier(params, name='classifier'):\n",
    "    \n",
    "    \"\"\"Helper function to save the classifer\n",
    "    \n",
    "    Parameters:\n",
    "        • params - a dict where the values are the classifier and all the parameters used to generate it\n",
    "        • name - name of the classifier to be used in the file name\"\"\"\n",
    "    \n",
    "    with open('model/{}.p'.format(name), 'wb') as f:\n",
    "        pickle.dump(params, f)    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "params = {'classifier': svm,\n",
    "    'scaler': scaler,\n",
    "    'color_space': color_space,\n",
    "    'orient': orient,\n",
    "    'pix_per_cell': pix_per_cell,\n",
    "    'cell_per_block': cell_per_block,\n",
    "    'hog_channel': hog_channel,\n",
    "    'hog_clahe': hog_clahe,\n",
    "    'spatial_size': spatial_size,\n",
    "    'hist_bins': hist_bins,\n",
    "    'spatial_feat': spatial_feat,\n",
    "    'hist_feat': hist_feat,\n",
    "    'hog_feat': hog_feat,\n",
    "}\n",
    "\n",
    "save_classifier(params, 'LinearSVM_Full')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "---\n",
    "\n",
    "## Vehicle detection\n",
    "\n",
    "### 0. Load the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_classifier(name='classifier'):\n",
    "    \n",
    "    \"\"\"Helper function to load a classifier\n",
    "    \n",
    "    Parameters:\n",
    "        • name - name of the classifer file without the extension or path\n",
    "        \n",
    "    Returns:\n",
    "        A dict where the values are the classifer and all the parameters used to generate it\n",
    "        loaded from the file model/`name`.p\"\"\"\n",
    "    \n",
    "    with open('model/{}.p'.format(name), 'rb') as f:\n",
    "        params = pickle.load(f)\n",
    "        \n",
    "    return params\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load the classifier\n",
    "params = load_classifier('LinearSVM_Full')\n",
    "\n",
    "svm = params['classifier']\n",
    "scaler = params['scaler']\n",
    "color_space = params['color_space']\n",
    "orient = params['orient']\n",
    "pix_per_cell = params['pix_per_cell']\n",
    "cell_per_block = params['cell_per_block']\n",
    "hog_channel = params['hog_channel']\n",
    "hog_clahe = params['hog_clahe']\n",
    "spatial_size = params['spatial_size']\n",
    "hist_bins = params['hist_bins']\n",
    "spatial_feat = params['spatial_feat']\n",
    "hist_feat = params['hist_feat']\n",
    "hog_feat = params['hog_feat']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Search for matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def draw_boxes(img, boxes, color=(0, 0, 255), thickness=6):\n",
    "    \n",
    "    \"\"\"Helper function to draw boxes on an image.\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image\n",
    "        • boxes - list of bounding boxes ((x_min, y_min), (x_max, y_max))\n",
    "        • color - a tuple of RGB values [0, 255]\n",
    "        • thickness - thickness of the lines to draw\n",
    "        \n",
    "    Returns:\n",
    "        The image with the boxes drawn on top of it\"\"\"\n",
    "    \n",
    "    draw_img = np.copy(img)\n",
    "    \n",
    "    for box in boxes:\n",
    "        point_min, point_max = box\n",
    "        \n",
    "        cv2.rectangle(draw_img, point_min, point_max, color, thickness)\n",
    "        \n",
    "    return draw_img\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def potential_cars(img, y_min, y_max, scale, svc, X_scaler, to_color_space, from_color_space, orient, pix_per_cell, \n",
    "                   cell_per_block, hog_channel, hog_clahe, spatial_size, hist_bins):\n",
    "    \n",
    "    \"\"\"Using a sliding window determine areas of the image representing potential cars using a classifier and some \n",
    "    parameters to generate the features to feed into it.\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image\n",
    "        • y_min - minimum y value in image to consider\n",
    "        • y_max - maximum y value in image to consider\n",
    "        • scale - scale of the sliding window to use\n",
    "        • X_scaler - feature scaler used on the training features for the classifier\n",
    "        • to_color_space - desired color space to extract features\n",
    "        • from_color_space - original color space of the image\n",
    "        • orient - number of orientations for HOG features\n",
    "        • pix_per_cell - cell size over which each gradient histogram is computed\n",
    "        • cell_per_block - specifies the local area over which the histogram counts in a given cell will be normalized\n",
    "        • hog_channel - image channel to apply the Histogram of Oriented Gradient (HOG)\n",
    "        • hog_clahe - boolean to use the Contrast Limited Adaptive Histogram Equalization for the HOG input\n",
    "        • spatial_size - size for spacial binning of color features\n",
    "        • hist_bins - number of bins for the color histogram features\n",
    "    \n",
    "    Returns:\n",
    "        A list of windows that matched a car\"\"\"\n",
    "    \n",
    "    # define the region of the image to search\n",
    "    search_img = img[y_min:y_max, :, :]\n",
    "    \n",
    "    # convert to the desired color space\n",
    "    search_img = convert_color(search_img, to_color_space, from_color_space)\n",
    "    \n",
    "    # if the scale is not 1, resize the search image (instead of the sliding window)\n",
    "    if scale != 1:\n",
    "        height, width, channels = search_img.shape\n",
    "        search_img = cv2.resize(search_img, (np.int(width / scale), np.int(height / scale)))\n",
    "    \n",
    "    height, width, channels = search_img.shape\n",
    "\n",
    "    # Define blocks and steps as above\n",
    "    nxblocks = (width // pix_per_cell) - 1\n",
    "    nyblocks = (height // pix_per_cell) - 1 \n",
    "    nfeat_per_block = orient * cell_per_block ** 2\n",
    "\n",
    "    # 64 was the orginal sampling rate, with 8 cells and 8 pix per cell\n",
    "    window = 64\n",
    "    nblocks_per_window = (window // pix_per_cell)-1 \n",
    "    cells_per_step = 2  # Instead of overlap, define how many cells to step\n",
    "    nxsteps = (nxblocks - nblocks_per_window) // cells_per_step\n",
    "    nysteps = (nyblocks - nblocks_per_window) // cells_per_step\n",
    "    \n",
    "    # Compute individual channel HOG features for the entire image\n",
    "    hogs = []\n",
    "    \n",
    "    hog_img = np.copy(search_img)\n",
    "\n",
    "    if hog_clahe:\n",
    "        hog_img[:, :, 0] = CLAHE(hog_img, from_color_space)\n",
    "        hog_channel = (hog_channel == 'ALL') and hog_channel or 0\n",
    "        plt.figure()\n",
    "        plt.imshow(convert_color(hog_img, 'RGB', 'YCrCb'))\n",
    "        \n",
    "    if hog_channel == 'ALL':\n",
    "        for c in range(3):\n",
    "            hogs.append(get_hog_features(hog_img[:, :, c], orient, pix_per_cell, cell_per_block, \n",
    "                                         feature_vec=False))\n",
    "    else:\n",
    "        hogs.append(get_hog_features(hog_img, orient, pix_per_cell, cell_per_block, feature_vec=False))\n",
    "    \n",
    "    bounding_boxes = []\n",
    "    \n",
    "    for xb in range(nxsteps):\n",
    "        for yb in range(nysteps):\n",
    "            ypos = yb * cells_per_step\n",
    "            xpos = xb * cells_per_step\n",
    "            # Extract HOG for this patch\n",
    "            hog_feats = [h[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel() for h in hogs]\n",
    "            hog_features = np.hstack(hog_feats)\n",
    "\n",
    "            xleft = xpos*pix_per_cell\n",
    "            ytop = ypos*pix_per_cell\n",
    "\n",
    "            # Extract the image patch\n",
    "            subimg = cv2.resize(search_img[ytop:ytop + window, xleft:xleft + window], (64,64))\n",
    "          \n",
    "            # Get color features\n",
    "            spatial_features = bin_spatial(subimg, size=spatial_size)\n",
    "            hist_features = color_hist(subimg, nbins=hist_bins)\n",
    "\n",
    "            # Scale features and make a prediction\n",
    "            test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features))\n",
    "                                               .reshape(1, -1))\n",
    "            test_prediction = svc.predict(test_features)\n",
    "            \n",
    "            if test_prediction == 1:\n",
    "                xbox_left = np.int(xleft * scale)\n",
    "                ytop_draw = np.int(ytop * scale)\n",
    "                win_draw = np.int(window * scale)\n",
    "                bounding_boxes.append(((xbox_left, ytop_draw + y_min), \n",
    "                                       (xbox_left + win_draw, ytop_draw + win_draw + y_min)))\n",
    "                \n",
    "    return bounding_boxes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_min = 400\n",
    "y_max = 656\n",
    "scales = [1.0, 1.5]\n",
    "\n",
    "test_imgs = glob.glob('test_images/*.jpg')\n",
    "\n",
    "all_bboxes = []\n",
    "\n",
    "for test_img in test_imgs:\n",
    "    img = cv2.imread(test_img)\n",
    "    \n",
    "    bboxes = []\n",
    "    for scale in scales:\n",
    "        scaled_bboxes = potential_cars(img, y_min, y_max, scale, svm, scaler, color_space, 'BGR', orient, \n",
    "                                       pix_per_cell, cell_per_block, hog_channel, hog_clahe, spatial_size, hist_bins) \n",
    "        bboxes.extend(scaled_bboxes)\n",
    "        \n",
    "    all_bboxes.append(bboxes)\n",
    "    detected_img = draw_boxes(convert_color(img, 'RGB'), bboxes)\n",
    "    plt.figure()\n",
    "    plt.imshow(detected_img);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Convert to a heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def labels_to_bboxes(labels):\n",
    "    \n",
    "    \"\"\"Helper function to convert labels from a heatmap into smallest fitting bounding boxes\n",
    "    \n",
    "    Parameters:\n",
    "        • labels - labels as returned by the scipy.ndimage.measurements.label function\n",
    "        \n",
    "    Returns:\n",
    "        A list of bounding boxes\"\"\"\n",
    "    \n",
    "    bboxes = []\n",
    "    \n",
    "    for label in range(1, labels[1] + 1):\n",
    "\n",
    "        # find pixels with each car_number label value\n",
    "        nonzero = (labels[0] == label).nonzero()\n",
    "        \n",
    "        # identify x and y values of those pixels\n",
    "        nonzeroy = np.array(nonzero[0])\n",
    "        nonzerox = np.array(nonzero[1])\n",
    "        \n",
    "        # define a bounding box based on min/max x and y\n",
    "        bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))\n",
    "        \n",
    "        # add to the list of bounding boxes\n",
    "        bboxes.append(bbox)\n",
    "        \n",
    "    return bboxes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def heatmapify(img, bboxes, threshold=None):\n",
    "    \n",
    "    \"\"\"Create a heatmap in the image based on a list of bounding boxes\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image\n",
    "        • bboxes - list of bounding boxes of potential matches\n",
    "        • threshold - threshold to filter out false positives (integer or `None`)\n",
    "        \n",
    "    Returns:\n",
    "        A tuple of the heatmap image and a list of bounding boxes of hot areas\"\"\"\n",
    "    \n",
    "    # create an empty heatmap\n",
    "    heatmap = np.zeros_like(img[:, :, 0]).astype(np.float)\n",
    "    \n",
    "    # iterate through boxes\n",
    "    for bbox in bboxes:\n",
    "        \n",
    "        # extract minimum and maximum points of the bounding boxes\n",
    "        point_min, point_max = bbox\n",
    "        \n",
    "        # increment pixels in the heatmap for all pixels within the bounding box\n",
    "        heatmap[point_min[1]:point_max[1], point_min[0]:point_max[0]] += 1\n",
    "    \n",
    "    # if there is a threshold, apply it\n",
    "    if threshold is not None:\n",
    "        heatmap[heatmap <= threshold] = 0\n",
    "    \n",
    "    # clip the heat map for visualizing purposes\n",
    "    heatmap = np.clip(heatmap, 0, 255)\n",
    "    \n",
    "    # create a list of boxes for remaining hot areas\n",
    "    labels = label(heatmap)\n",
    "    final_bboxes = labels_to_bboxes(labels)\n",
    "    \n",
    "    return heatmap, final_bboxes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for test_img, bboxes in zip(test_imgs, all_bboxes):\n",
    "    img = cv2.imread(test_img)\n",
    "    heatmap, final_bboxes = heatmapify(img, bboxes, threshold=1)\n",
    "    detected_img = draw_boxes(convert_color(img, 'RGB'), final_bboxes)\n",
    "\n",
    "    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))\n",
    "    ax1.imshow(heatmap, cmap='hot')\n",
    "    ax2.imshow(detected_img)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "# Pipeline\n",
    "\n",
    "Put all the vehicle detection pieces together into a single function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_cars(img, classifier, scaler, params=[]):\n",
    "    \n",
    "    \"\"\"Find the cars in an image and overlay bounding boxes on them.\n",
    "    \n",
    "    Parameters:\n",
    "        • img - input image to search\n",
    "        • classifier - classifier to use to search for cars\n",
    "        • scaler - scaler used on the feature vector\n",
    "        • params - dictionary of parameters to control the search (see `potential_cars` and `heatmapify` funcs)\n",
    "        \n",
    "    Returns:\n",
    "        An image with the cars marked by boxes\"\"\"\n",
    "    \n",
    "    \n",
    "    # extract parameters from the params dictionary or use default values\n",
    "    to_color_space = params.get('to_color_space', 'RGB')\n",
    "    from_color_space = params.get('from_color_space', 'RGB')\n",
    "    y_min = params.get('y_min', 400)\n",
    "    y_max = params.get('y_max', 656)\n",
    "    scales = params.get('scales', [1.5])\n",
    "    orient = params.get('orient', 9)\n",
    "    pix_per_cell = params.get('pix_per_cell', 8)\n",
    "    cell_per_block = params.get('cell_per_block', 2)\n",
    "    hog_channel = params.get('hog_channel', 0)\n",
    "    hog_clahe = params.get('hog_clahe', True)\n",
    "    spatial_size = params.get('spatial_size', (32, 32))\n",
    "    hist_bins = params.get('hist_bins', 32)\n",
    "    spatial_feat = params.get('spatial_feat', True)\n",
    "    hist_feat = params.get('hist_feat', True)\n",
    "    hog_feat = params.get('hog_feat', True)\n",
    "    threshold = params.get('threshold', 1)\n",
    "    prev_frame_cnt = params.get('prev_frame_cnt', 0)\n",
    "    prev_frames_heat = params.get('prev_frames_heat', [])\n",
    "    \n",
    "    # get the bounding boxes of matches at all scales\n",
    "    bboxes = []\n",
    "    for scale in scales:\n",
    "        scaled_bboxes = potential_cars(img, y_min, y_max, scale, svm, scaler, to_color_space, from_color_space, \n",
    "                                       orient, pix_per_cell, cell_per_block, hog_channel, hog_clahe, spatial_size, \n",
    "                                       hist_bins) \n",
    "        bboxes.extend(scaled_bboxes)\n",
    "\n",
    "    # take previous frames into account by merging their detected bounding boxes into the inputs for the heatmap\n",
    "    if prev_frame_cnt > 0:\n",
    "        # threshold should increase linearly with the number of previous frames being added to the heatmap\n",
    "        threshold += len(prev_frames_heat)\n",
    "        \n",
    "        # add this frame's bounding boxes to the list\n",
    "        prev_frames_heat.append(bboxes)\n",
    "        \n",
    "        # combine all bounding boxes into one list (instead of a list of lists)\n",
    "        bboxes = list(itertools.chain.from_iterable(prev_frames_heat))\n",
    "        \n",
    "        # if we hit the limit of frames saved, remove the first one (FIFO)\n",
    "        if len(prev_frames_heat) > prev_frame_cnt:\n",
    "            prev_frames_heat = prev_frames_heat[1:]\n",
    "            \n",
    "        # reset the dictionary value for the previous frames heat\n",
    "        params['prev_frames_heat'] = prev_frames_heat\n",
    "        \n",
    "    # create a heatmap based on the bounding boxes and threshold it\n",
    "    heatmap, final_bboxes = heatmapify(img, bboxes, threshold=threshold)\n",
    "\n",
    "    # create a copy of the image in RGB to draw on\n",
    "    detected_img = convert_color(img, 'RGB', from_color_space)\n",
    "    \n",
    "    # draw the final bounding boxes on the image\n",
    "    detected_img = draw_boxes(detected_img, final_bboxes)\n",
    "    \n",
    "    return detected_img\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load the classifier\n",
    "params = load_classifier('LinearSVM_Full')\n",
    "\n",
    "params['to_color_space'] = params['color_space']\n",
    "params['from_color_space'] = 'BGR'\n",
    "params['y_min'] = 400\n",
    "params['y_max'] = 656\n",
    "params['scales'] = [1.0, 1.5]\n",
    "params['threshold'] = 1\n",
    "\n",
    "# create list of test images\n",
    "test_imgs = glob.glob('test_images/*.jpg')\n",
    "\n",
    "# cycle through the test images\n",
    "for test_img in test_imgs:\n",
    "    \n",
    "    # read in the image\n",
    "    img = cv2.imread(test_img)\n",
    "    \n",
    "    # run the pipeline\n",
    "    detected_img = find_cars(img, params['classifier'], params['scaler'], params)\n",
    "    \n",
    "    # display the results\n",
    "    plt.figure()\n",
    "    plt.imshow(detected_img);\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "# Test on Videos\n",
    "\n",
    "Let's try the pipeline on some video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import everything needed to edit/save/watch video clips\n",
    "from moviepy.editor import VideoFileClip\n",
    "from IPython.display import HTML\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. Load the classifier and set up params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the classifier\n",
    "params = load_classifier('LinearSVM_Full')\n",
    "\n",
    "params['to_color_space'] = params['color_space']\n",
    "params['from_color_space'] = 'RGB'\n",
    "params['y_min'] = 400\n",
    "params['y_max'] = 656\n",
    "params['scales'] = [1.0, 1.5]\n",
    "params['threshold'] = 1\n",
    "\n",
    "classifier = params['classifier']\n",
    "scaler = params['scaler']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. test_video.mp4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set parameters for the find_cars pipeline\n",
    "params['to_color_space'] = 'YCrCb'\n",
    "# use the previous 3 frames when deciding which matches are valid\n",
    "params['prev_frame_cnt'] = 3\n",
    "params['prev_frames_heat'] = []\n",
    "params['threshold'] = 3\n",
    "\n",
    "# run test video through the pipeline\n",
    "test_output = 'test_output.mp4'\n",
    "clip1 = VideoFileClip(\"test_video.mp4\")\n",
    "test_clip = clip1.fl_image(lambda x: find_cars(x, classifier, scaler, params)) #NOTE: this function expects color images!!\n",
    "%time test_clip.write_videofile(test_output, audio=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "HTML(\"\"\"\n",
    "<video width=\"960\" height=\"540\" controls>\n",
    "  <source src=\"{0}\">\n",
    "</video>\n",
    "\"\"\".format(test_output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. project_video.mp4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set parameters for the find_cars pipeline\n",
    "params['to_color_space'] = 'YCrCb'\n",
    "# use the previous 3 frames when deciding which matches are valid\n",
    "params['prev_frame_cnt'] = 3\n",
    "params['prev_frames_heat'] = []\n",
    "params['threshold'] = 3\n",
    "\n",
    "# run test video through the pipeline\n",
    "project_output = 'project_output.mp4'\n",
    "clip2 = VideoFileClip(\"project_video.mp4\")\n",
    "project_clip = clip2.fl_image(lambda x: find_cars(x, classifier, scaler, params)) #NOTE: this function expects color images!!\n",
    "%time project_clip.write_videofile(project_output, audio=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "HTML(\"\"\"\n",
    "<video width=\"960\" height=\"540\" controls>\n",
    "  <source src=\"{0}\">\n",
    "</video>\n",
    "\"\"\".format(project_output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}