face_verf_arl3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XyK6R3drQkJ-"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import util.util as util\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import glob\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from scipy.optimize import brentq\n",
    "from scipy.interpolate import interp1d\n",
    "\n",
    "from tqdm import tqdm\n",
    "import os.path as path\n",
    "import torch.nn.functional as F\n",
    "from models import networks\n",
    "\n",
    "from PIL import Image\n",
    "import random\n",
    "import numpy as np\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import csv\n",
    "\n",
    "from util.vggface import VGGFace as VggFace\n",
    "from options.test_options import TestOptions\n",
    "from models import create_model\n",
    "\n",
    "import ipdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XyK6R3drQkJ-"
   },
   "outputs": [],
   "source": [
    "# os.environ['CUDA_VISIBLE_DEVICES']='1'\n",
    "gpu_ids = ['cuda:0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gallery_root = '../../dataset/odin_data/Vis_128/'\n",
    "# probe_root = '../../dataset/odin_data/Polar_128_rec/'\n",
    "probe_root = '../../dataset/odin_data/Polar_16/'\n",
    "split_dir = '../../dataset/odin_data/splits/'\n",
    "\n",
    "run_rec_metrics = True\n",
    "run_id = True\n",
    "\n",
    "args = \"--dataset_mode aligned_arl3 --A_mode Polar --no_pose\"\n",
    "# args += \" --dont_save_metrics\"\n",
    "# args += \" --model pix2pix\"\n",
    "# args += \" --model pix2pix_conf --netG SAGAN\"\n",
    "args += \" --model pix2pix_conf\"\n",
    "\n",
    "exp = '001_axial_arl3_S0'\n",
    "\n",
    "split = int(exp[-1])\n",
    "args += \" --split {}\".format(split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test PSNR and SSIM\n",
    "if run_rec_metrics:\n",
    "    cmd = \"python test_metrics.py --name {} --dir_A {} --epoch {} {}\".format(exp, probe_root, 'best', args)\n",
    "    !{cmd}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test PSNR and SSIM\n",
    "if run_rec_metrics:\n",
    "    cmd = \"python test_metrics.py --name {} --dir_A {} --epoch {} {}\".format(exp, probe_root, 'best_ssim', args)\n",
    "    !{cmd}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test PSNR and SSIM\n",
    "if run_rec_metrics and run_id:\n",
    "    cmd = \"python test_metrics.py --name {} --dir_A {} --epoch {} {}\".format(exp, probe_root, 'best_id', args)\n",
    "    !{cmd}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XyK6R3drQkJ-"
   },
   "outputs": [],
   "source": [
    "model = VggFace()\n",
    "model.load_state_dict(torch.load(os.path.join('./checkpoints/', \"vggface.pth\")))\n",
    "model.to(gpu_ids[0])\n",
    "model.eval()\n",
    "    \n",
    "data_transforms = transforms.Compose([\n",
    "        transforms.Resize([128, 128]),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "#         transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ])\n",
    "\n",
    "pix2pix_transform = transforms.Compose([\n",
    "#         transforms.Resize([128, 128]),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "#         transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ])\n",
    "\n",
    "# vgg_transforms = transforms.Compose([\n",
    "#         transforms.Resize([128, 128]),\n",
    "#         transforms.ToTensor(),\n",
    "# #         transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "#         transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "#     ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IPmIS4jSQkKH"
   },
   "outputs": [],
   "source": [
    "def display_result(fpr, tpr, thresholds, roc_auc):\n",
    "    eer = brentq(lambda x : 1. - x - interp1d(fpr, tpr)(x), 0., 1.)\n",
    "    thresh = interp1d(fpr, thresholds)(eer)\n",
    "    f = interp1d(fpr, tpr)\n",
    "    \n",
    "    l1 = 'AUC: %f ' % (roc_auc)\n",
    "    l2 = 'EER: %f ' % (eer)\n",
    "    l3 = \"FAR=0.01: %f \" % (f(0.01))\n",
    "    l4 = \"FAR=0.05: %f \" % (f(0.05))\n",
    "    \n",
    "    print(l1, l2, l3, l4, sep='\\n')    \n",
    "    ret = [roc_auc, eer, f(0.01), f(0.05)]\n",
    "    ret = [round(x*100,2) for x in ret]\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_id(x):\n",
    "    return x.split('_')[2][:-1]\n",
    "\n",
    "def remove_pose(paths):\n",
    "    paths = [x for x in paths if '_p_' not in x]\n",
    "    print('Removed pose images')\n",
    "    return paths\n",
    "\n",
    "def compute_probe():\n",
    "    print('Computing embeddings of', probe_imgs)\n",
    "    \n",
    "    fp = open(probe_imgs).readlines()\n",
    "    if opt.no_pose:\n",
    "        fp = remove_pose(fp)\n",
    "    count_num = len(fp)\n",
    "    outputs_probe = np.zeros((count_num,25088))\n",
    "    namelist_probe = []\n",
    "    \n",
    "    for cnt, line in enumerate(tqdm(fp)):\n",
    "        A_data = line.rstrip().replace('V1',opt.A_mode)\n",
    "        A_path = os.path.join(probe_root, A_data)\n",
    "        name = get_id(A_data)\n",
    "\n",
    "        A_img = Image.open(A_path)    \n",
    "        var_A = pix2pix_transform(A_img).unsqueeze(0)\n",
    "        var_A = var_A.to(gpu_ids[0])\n",
    "        d1_A = model_A(var_A)\n",
    "\n",
    "        feat_probe = model(F.interpolate(d1_A, size=(224, 224), mode='bilinear', align_corners=True))[-1]\n",
    "        outputs_probe[cnt,:] = feat_probe.view(1,-1).cpu().data[0]\n",
    "        namelist_probe.append(name)\n",
    "\n",
    "    return outputs_probe, namelist_probe\n",
    "\n",
    "def compute_gallery():\n",
    "    print('Computing embeddings of', gallery_imgs)\n",
    "    \n",
    "    fp = open(gallery_imgs).readlines()\n",
    "    if opt.no_pose:\n",
    "        fp = remove_pose(fp)\n",
    "    count_num = len(fp)\n",
    "    outputs_gallery = np.zeros((count_num,25088))\n",
    "    namelist_gallery = []\n",
    "    \n",
    "    for cnt, line in enumerate(tqdm(fp)):\n",
    "        B_data = line.rstrip()\n",
    "        B_path = os.path.join(gallery_root, B_data)\n",
    "        name = get_id(B_data)\n",
    "\n",
    "        B_img = Image.open(B_path).convert('RGB')\n",
    "        var_B = data_transforms(B_img).unsqueeze(0)\n",
    "\n",
    "        var_B = var_B.to(gpu_ids[0])\n",
    "        feat_gallery = model(F.interpolate(var_B, size=(224, 224), mode='bilinear', align_corners=True))[-1]\n",
    "        outputs_gallery[cnt,:] = feat_gallery.view(1,-1).cpu().data[0]\n",
    "        namelist_gallery.append(name)\n",
    "\n",
    "    return outputs_gallery, namelist_gallery\n",
    "\n",
    "def compute_result():\n",
    "    similary_matrix = np.asarray(cosine_similarity(outputs_probe,outputs_gallery))\n",
    "    gt_matrix = np.zeros([similary_matrix.shape[0], similary_matrix.shape[1]])\n",
    "    for i in range(similary_matrix.shape[0]):\n",
    "        for j in range(similary_matrix.shape[1]):\n",
    "            if namelist_probe[i] == namelist_gallery[j]:\n",
    "                gt_matrix[i, j] = 1\n",
    "            else:\n",
    "                gt_matrix[i, j] = -1\n",
    "    \n",
    "    # save result matrix\n",
    "    gallery_id = os.path.splitext(gallery_imgs.split(\"/\")[-1])[0]\n",
    "    probe_id = os.path.splitext(probe_imgs.split(\"/\")[-1])[0]\n",
    "    \n",
    "    pd.DataFrame(gt_matrix).to_csv(os.path.join(sim_matrix_folder, \"gt_\" + gallery_id + \"_\" + probe_id + \".csv\"))\n",
    "    pd.DataFrame(similary_matrix).to_csv(os.path.join(sim_matrix_folder, \"pred_\" + gallery_id + \"_\" + probe_id + \".csv\"))\n",
    "    \n",
    "    print(gallery_id, probe_id)\n",
    "    print(\"Gallery size:\", len(namelist_gallery))\n",
    "    print(\"Probe size:\", len(namelist_probe))\n",
    "    \n",
    "    # compute scores                 \n",
    "    fpr, tpr, thresholds = roc_curve(gt_matrix.reshape(-1), similary_matrix.reshape(-1))\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    roc_data[(gallery_id, probe_id)] = (fpr, tpr, thresholds)\n",
    "    \n",
    "    row = display_result(fpr, tpr, thresholds, roc_auc)\n",
    "    if not opt.dont_save_metrics:\n",
    "        with open(verf_file,'a') as f:\n",
    "            csvwriter = csv.writer(f)\n",
    "            row = ['{},{}'.format(gallery_id, probe_id)] + row\n",
    "            csvwriter.writerow(row)\n",
    "\n",
    "    plt.imshow(gt_matrix)\n",
    "    plt.show()\n",
    "    plt.imshow(similary_matrix)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_args = \"\"\n",
    "if len(args) > 0:\n",
    "    split_args = args.split(\" \")\n",
    "opt = TestOptions().parse(split_args)\n",
    "opt.name = exp\n",
    "opt.dir_A = probe_root\n",
    "\n",
    "mxroot = os.path.join(opt.results_dir, exp)\n",
    "verf_file = os.path.join(mxroot, 'all_metrics.csv')\n",
    "\n",
    "# gallery data remains same\n",
    "gallery_ids = [opt.split]\n",
    "gallery_data = {}\n",
    "gallery_dir = os.path.join(opt.checkpoints_dir, opt.dataset_mode.split('_')[-1])\n",
    "os.makedirs(gallery_dir, exist_ok=True)\n",
    "gallery_feats_path = os.path.join(gallery_dir, 'gallery_feats_{}_s{}.npy'.format(opt.A_mode, opt.split))\n",
    "if os.path.exists(gallery_feats_path):\n",
    "    print('Loading gallery data')\n",
    "    gallery_data = np.load(gallery_feats_path, allow_pickle=True).item()\n",
    "else:\n",
    "    for id_ in gallery_ids:\n",
    "        gallery_imgs = os.path.join(split_dir, 'gallery_{}.txt'.format(id_))\n",
    "        gallery_data[id_] = compute_gallery()\n",
    "    print('Saving gallery data')\n",
    "    np.save(gallery_feats_path, gallery_data)\n",
    "    \n",
    "probe_ids = [opt.split]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dCKe5fVLQkKI",
    "outputId": "ffabeb93-39f3-40e3-d268-eae7f5d825dc"
   },
   "outputs": [],
   "source": [
    "opt.epoch = 'best'\n",
    "\n",
    "sim_matrix_folder = os.path.join(mxroot, 'similarity_matrix_' + opt.epoch)\n",
    "os.makedirs(sim_matrix_folder, exist_ok=True)\n",
    "\n",
    "full_model = create_model(opt)\n",
    "full_model.setup(opt)\n",
    "model_A = full_model.netG\n",
    "model_A.eval()\n",
    "\n",
    "if not opt.dont_save_metrics:\n",
    "    now = time.strftime(\"%c\")\n",
    "    with open(verf_file,'a') as f:\n",
    "        csvwriter = csv.writer(f)\n",
    "        rows = [['Verification metrics', now, 'Epoch_{}'.format(opt.epoch)], \n",
    "                ['Protocol', 'AUC', 'EER', 'FAR=1%','FAR=5%']]\n",
    "        csvwriter.writerows(rows)\n",
    "\n",
    "probe_data = {}\n",
    "for id_ in probe_ids:\n",
    "    probe_imgs = os.path.join(split_dir, 'probe_{}.txt'.format(id_))\n",
    "    probe_data[id_] = compute_probe()\n",
    "\n",
    "roc_data = {}    \n",
    "# compute result for all combinations of gallery and probe\n",
    "for gid in gallery_data.keys():\n",
    "    for pid in probe_data.keys():\n",
    "        gallery_imgs = os.path.join(split_dir, 'gallery_{}.txt'.format(gid))\n",
    "        probe_imgs = os.path.join(split_dir, 'probe_{}.txt'.format(pid))\n",
    "        \n",
    "        outputs_gallery, namelist_gallery = gallery_data[gid]\n",
    "        outputs_probe, namelist_probe = probe_data[pid]\n",
    "        compute_result()\n",
    "        \n",
    "roc_path = os.path.join(mxroot, 'roc_' + opt.epoch + '.npy')        \n",
    "np.save(roc_path, roc_data)           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt.epoch = 'best_ssim'\n",
    "\n",
    "sim_matrix_folder = os.path.join(mxroot, 'similarity_matrix_' + opt.epoch)\n",
    "os.makedirs(sim_matrix_folder, exist_ok=True)\n",
    "\n",
    "full_model = create_model(opt)\n",
    "full_model.setup(opt)\n",
    "model_A = full_model.netG\n",
    "model_A.eval()\n",
    "\n",
    "if not opt.dont_save_metrics:\n",
    "    now = time.strftime(\"%c\")\n",
    "    with open(verf_file,'a') as f:\n",
    "        csvwriter = csv.writer(f)\n",
    "        rows = [['Verification metrics', now, 'Epoch_{}'.format(opt.epoch)], \n",
    "                ['Protocol', 'AUC', 'EER', 'FAR=1%','FAR=5%']]\n",
    "        csvwriter.writerows(rows)\n",
    "\n",
    "probe_data = {}\n",
    "for id_ in probe_ids:\n",
    "    probe_imgs = os.path.join(split_dir, 'probe_{}.txt'.format(id_))\n",
    "    probe_data[id_] = compute_probe()\n",
    "    \n",
    "roc_data = {}    \n",
    "# compute result for all combinations of gallery and probe\n",
    "for gid in gallery_data.keys():\n",
    "    for pid in probe_data.keys():\n",
    "        gallery_imgs = os.path.join(split_dir, 'gallery_{}.txt'.format(gid))\n",
    "        probe_imgs = os.path.join(split_dir, 'probe_{}.txt'.format(pid))\n",
    "        \n",
    "        outputs_gallery, namelist_gallery = gallery_data[gid]\n",
    "        outputs_probe, namelist_probe = probe_data[pid]\n",
    "        compute_result()\n",
    "        \n",
    "roc_path = os.path.join(mxroot, 'roc_' + opt.epoch + '.npy')        \n",
    "np.save(roc_path, roc_data)           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt.epoch = 'best_id'\n",
    "\n",
    "full_model = create_model(opt)\n",
    "full_model.setup(opt)\n",
    "model_A = full_model.netG\n",
    "model_A.eval()\n",
    "\n",
    "sim_matrix_folder = os.path.join(mxroot, 'similarity_matrix_' + opt.epoch)\n",
    "os.makedirs(sim_matrix_folder, exist_ok=True)\n",
    "\n",
    "if not opt.dont_save_metrics:\n",
    "    now = time.strftime(\"%c\")\n",
    "    with open(verf_file,'a') as f:\n",
    "        csvwriter = csv.writer(f)\n",
    "        rows = [['Verification metrics', now, 'Epoch_{}'.format(opt.epoch)], \n",
    "                ['Protocol', 'AUC', 'EER', 'FAR=1%','FAR=5%']]\n",
    "        csvwriter.writerows(rows)\n",
    "\n",
    "probe_data = {}\n",
    "for id_ in probe_ids:\n",
    "    probe_imgs = os.path.join(split_dir, 'probe_{}.txt'.format(id_))\n",
    "    probe_data[id_] = compute_probe()\n",
    "    \n",
    "roc_data = {}    \n",
    "# compute result for all combinations of gallery and probe\n",
    "for gid in gallery_data.keys():\n",
    "    for pid in probe_data.keys():\n",
    "        gallery_imgs = os.path.join(split_dir, 'gallery_{}.txt'.format(gid))\n",
    "        probe_imgs = os.path.join(split_dir, 'probe_{}.txt'.format(pid))\n",
    "        \n",
    "        outputs_gallery, namelist_gallery = gallery_data[gid]\n",
    "        outputs_probe, namelist_probe = probe_data[pid]\n",
    "        compute_result()\n",
    "        \n",
    "roc_path = os.path.join(mxroot, 'roc_' + opt.epoch + '.npy')        \n",
    "np.save(roc_path, roc_data)           "
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "odin3_cpf_pix2pix_testing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}