ImportModels.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import GAN\n",
    "trained using  https://github.com/alinlab/Confident_classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "import utils.confident_classifier as cc\n",
    "\n",
    "\n",
    "folder = '../Confident_classifier/SavedModels/'\n",
    "\n",
    "# MNIST\n",
    "file = 'MNIST_epochs100_lr0.0002_classifier.pth'\n",
    "model = cc.LeNet()\n",
    "model.load_state_dict(torch.load(folder + file))\n",
    "\n",
    "model.cpu()\n",
    "torch.save(model, 'SavedModels/other/gan/' + 'MNIST' + '.pth')\n",
    "\n",
    "\n",
    "# FMNIST\n",
    "file = 'FMNIST_epochs100_lr0.0002_beta1.0_type_VGG_classifier.pth'\n",
    "model = cc.vgg13(in_channels=1, num_classes=10)\n",
    "model.load_state_dict(torch.load(folder + file))\n",
    "\n",
    "model.cpu()\n",
    "torch.save(model, 'SavedModels/other/gan/' + 'FMNIST' + '.pth')\n",
    "\n",
    "\n",
    "# SVHN\n",
    "file = 'SVHN_epochs100_lr0.0002_beta1.0_type_VGG_classifier.pth'\n",
    "model = cc.vgg13(in_channels=3, num_classes=10)\n",
    "model.load_state_dict(torch.load(folder + file))\n",
    "\n",
    "model.cpu()\n",
    "torch.save(model, 'SavedModels/other/gan/' + 'SVHN' + '.pth')\n",
    "\n",
    "\n",
    "# CIFAR10\n",
    "file = 'CIFAR10_epochs100_lr0.0002_beta1.0_type_VGG_classifier.pth'\n",
    "model = cc.vgg13(in_channels=3, num_classes=10)\n",
    "model.load_state_dict(torch.load(folder + file))\n",
    "\n",
    "model.cpu()\n",
    "torch.save(model, 'SavedModels/other/gan/' + 'CIFAR10' + '.pth')\n",
    "\n",
    "\n",
    "# CIFAR100\n",
    "file = 'CIFAR100_epochs100_lr0.0002_beta1.0_type_VGG_classifier.pth'\n",
    "model = cc.vgg13(in_channels=3, num_classes=100)\n",
    "model.load_state_dict(torch.load(folder + file))\n",
    "\n",
    "model.cpu()\n",
    "torch.save(model, 'SavedModels/other/gan/' + 'CIFAR100' + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import OE\n",
    "trained using https://github.com/hendrycks/outlier-exposure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import utils.hendrycks as oe\n",
    "import utils.dataloaders as dl\n",
    "\n",
    "\n",
    "folder = 'SavedModels/other/outlier-exposure/'\n",
    "device = torch.device('cpu')\n",
    "\n",
    "\n",
    "# MNIST\n",
    "file = '../outlier-exposure/MNIST/snapshots/oe_scratch/oe_scratch_epoch_99.pt'\n",
    "\n",
    "model = oe.MNIST_ConvNet()\n",
    "model.load_state_dict(torch.load(file, map_location = device))\n",
    "model.cpu()\n",
    "torch.save(model, folder + 'hendrycks_MNIST' + '.pth')\n",
    "\n",
    "\n",
    "# FMNIST\n",
    "file = '../outlier-exposure/FMNIST/snapshots/oe_scratch/oe_scratch_epoch_99.pt'\n",
    "\n",
    "#model = oe.WideResNet(40, 10, channels=1, dataset='FMNIST')\n",
    "model = oe.ResNet18(num_of_channels=1, num_classes=10, dataset='FMNIST')\n",
    "model.load_state_dict(torch.load(file, map_location = device))\n",
    "model.cpu()\n",
    "torch.save(model, folder + 'hendrycks_FMNIST' + '.pth')\n",
    "\n",
    "\n",
    "# SVHN\n",
    "file = '../outlier-exposure/SVHN/snapshots/oe_scratch/wrn_oe_scratch_epoch_99.pt'\n",
    "\n",
    "#model = wrn.WideResNet(16, 10, 4, 0.4, dataset='SVHN')\n",
    "model = oe.ResNet18(num_of_channels=3, num_classes=10, dataset='SVHN')\n",
    "model.load_state_dict(torch.load(file, map_location = device))\n",
    "model.cpu()\n",
    "torch.save(model, folder + 'hendrycks_SVHN' + '.pth')\n",
    "\n",
    "\n",
    "# CIFAR10\n",
    "file = '../outlier-exposure/CIFAR/snapshots/oe_scratch/cifar10_resnet_oe_scratch_epoch_99.pt'\n",
    "\n",
    "#model = wrn.WideResNet(40, 10, 2, 0.3)\n",
    "model = oe.ResNet18(num_of_channels=3, num_classes=10, dataset='CIFAR10')\n",
    "model.load_state_dict(torch.load(file, map_location = device))\n",
    "model.cpu()\n",
    "torch.save(model, folder + 'hendrycks_CIFAR10' + '.pth')\n",
    "\n",
    "\n",
    "# CIFAR100\n",
    "file = '../outlier-exposure/CIFAR/snapshots/oe_scratch/cifar100_resnet_oe_scratch_epoch_99.pt'\n",
    "\n",
    "#model = wrn.WideResNet(40, 10, 2, 0.3)\n",
    "model = oe.ResNet18(num_of_channels=3, num_classes=100, dataset='CIFAR100')\n",
    "model.load_state_dict(torch.load(file, map_location = device))\n",
    "model.cpu()\n",
    "torch.save(model, folder + 'hendrycks_CIFAR100' + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calibrate [ODIN](https://arxiv.org/abs/1706.02690)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CIFAR100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbed041f98>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4438>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4322d30>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbea6b90f0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbea6e8f28>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4d30>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4329e80>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbea6e8b38>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc23c8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4588>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4329198>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4518>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4d30>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c45f8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc2908>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc28d0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4080>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4329940>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4322ba8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9cf8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 383, in _recv\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd97f0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbed041e80>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbea6e8b70>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4329da0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9630>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd92b0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 383, in _recv\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc24a8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd94e0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9a90>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9fd0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbea6e8e48>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc23c8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4f28>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbed503f28>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd97b8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc2208>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 383, in _recv\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc20b8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbeabb4eb8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe4322e48>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe43291d0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9c18>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc24e0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9048>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbeabb43c8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9e48>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc2c18>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9668>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4f98>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbe3fc27b8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd2c4c4908>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbeabb41d0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd860>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd8d0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd7b8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fdb00>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd128>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd9e8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 383, in _recv\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd3c8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcb981fd7b8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcd25fd9940>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbeabb4e80>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fcbeabb4ef0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 383, in _recv\n",
      "    raise EOFError\n",
      "EOFError: \n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import model_params as params\n",
    "import utils.odin as odin\n",
    "\n",
    "import model_paths\n",
    "\n",
    "\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "out_seeds = True\n",
    "\n",
    "datasets = ['MNIST', 'FMNIST', 'SVHN', 'CIFAR10', 'CIFAR100']\n",
    "\n",
    "for dataset in datasets:\n",
    "    print(dataset)\n",
    "    model_params = params.params_dict[dataset](augm_flag=True)\n",
    "    path = model_paths.model_dict[dataset]().file_dict['Base']\n",
    "\n",
    "    model_params = params.params_dict[dataset]()\n",
    "\n",
    "    base_model = torch.load(path).to(device)\n",
    "    ODIN_model, _, _ = odin.grid_search_variables(base_model.to(device), model_params, \n",
    "                                                  device, out_seeds=out_seeds)\n",
    "\n",
    "    torch.save(ODIN_model.cpu(), 'SavedModels/other/odin/' + dataset + '_ODIN.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calibrate [Maha](https://arxiv.org/abs/1807.03888)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CIFAR100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f7722fe70f0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 493, in Client\n",
      "    answer_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 737, in answer_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f772334c898>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f7722fe78d0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f7723346b70>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 154, in recvfds\n",
      "    raise EOFError\n",
      "EOFError: \n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f7722fe7438>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import utils.dataloaders as dl\n",
    "\n",
    "import utils.single_maha as maha\n",
    "\n",
    "import model_params as params\n",
    "import model_paths\n",
    "\n",
    "\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "datasets = ['MNIST', 'FMNIST', 'SVHN', 'CIFAR10', 'CIFAR100']\n",
    "\n",
    "\n",
    "for dataset in datasets:\n",
    "    print(dataset)\n",
    "    model_params = params.params_dict[dataset](augm_flag=True)\n",
    "    path = model_paths.model_dict[dataset]().file_dict['Base']\n",
    "\n",
    "    pretrained = torch.load(path).state_dict()\n",
    "\n",
    "    if dataset=='MNIST':\n",
    "        model = maha.LeNetMadry()\n",
    "    elif dataset=='FMNIST':\n",
    "        model = maha.ResNet18(1, 10)\n",
    "    elif dataset in ['SVHN', 'CIFAR10']:\n",
    "        model = maha.ResNet18(3, 10)\n",
    "    elif dataset in ['CIFAR100']:\n",
    "        model = maha.ResNet18(3, 100)\n",
    "        \n",
    "    model.load_state_dict(pretrained)\n",
    "\n",
    "    maha_model = maha.Mahalanobis(model.to(device), model_params, device)\n",
    "    final_model = maha.ModelODIN(maha_model, model_params, device)\n",
    "\n",
    "    torch.save(final_model.cpu(), 'SavedModels/other/single_mahalanobis/' + dataset + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import [DE](http://papers.nips.cc/paper/7219-simple-and-scalable-predictive-uncertainty-estimation-using-deep-ensembles.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "import utils.models as models\n",
    "import utils.resnet_orig as resnet\n",
    "import utils.deep_ensemble as de\n",
    "\n",
    "\n",
    "folder = '../deep_ensembles/SavedModels/' \n",
    "\n",
    "\n",
    "datasets = ['MNIST', 'FMNIST', 'SVHN', 'CIFAR10', 'CIFAR100']\n",
    "\n",
    "\n",
    "for dataset in datasets:\n",
    "    model_list = []\n",
    "    M = 5\n",
    "\n",
    "    for m in range(M):\n",
    "        file = dataset + '_' + str(m) + '.pt'\n",
    "\n",
    "        if dataset=='MNIST':\n",
    "            model = models.LeNetMadry()\n",
    "        elif dataset=='FMNIST':\n",
    "            model = resnet.ResNet18(1, 10)\n",
    "        elif dataset in ['SVHN', 'CIFAR10']:\n",
    "            model = resnet.ResNet18(3, 10)\n",
    "        elif dataset in ['CIFAR100']:\n",
    "            model = resnet.ResNet18(3, 100)\n",
    "\n",
    "        model.load_state_dict(torch.load(folder + file))\n",
    "\n",
    "        model_list.append(model)\n",
    "\n",
    "    model = de.DeepEnsemble(model_list)\n",
    "\n",
    "    torch.save(model, 'SavedModels/other/deep_ensemble/' + dataset + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train [EDL](http://papers.nips.cc/paper/7580-evidential-deep-learning-to-quantify-classification-uncertainty.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import utils.edl as edl\n",
    "import utils.traintest as tt\n",
    "\n",
    "from importlib import reload\n",
    "\n",
    "import utils.dataloaders as dl\n",
    "\n",
    "import model_params\n",
    "\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "datasets = ['MNIST', 'FMNIST', 'SVHN', 'CIFAR10', 'CIFAR100']\n",
    "\n",
    "for dataset in datasets:\n",
    "    print(dataset + '\\n\\n\\n\\n')\n",
    "    params = model_params.params_dict[dataset](augm_flag=True)\n",
    "\n",
    "    if dataset=='MNIST':\n",
    "        base_model = edl.LeNet()\n",
    "    elif dataset=='FMNIST':\n",
    "        base_model = edl.ResNet18(1, 10)\n",
    "    elif dataset in ['SVHN', 'CIFAR10']:\n",
    "        base_model = edl.ResNet18(3, 10)\n",
    "    elif dataset in ['CIFAR100']:\n",
    "        base_model = edl.ResNet18(3, 100)\n",
    "\n",
    "    model = edl.EDL(base_model).to(device)\n",
    "\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "    for epoch in range(100):\n",
    "        if epoch+1 in [50,75,90]:\n",
    "            for group in optimizer.param_groups:\n",
    "                group['lr'] *= .1\n",
    "\n",
    "        trainloss, correct_train = edl.train_edl(model, device, params.train_loader,  \n",
    "                                                 optimizer, epoch, verbose=200)\n",
    "\n",
    "    model.eval()\n",
    "    model = model.to('cpu')\n",
    "\n",
    "    torch.save(model, 'SavedModels/other/edl/' + dataset + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train [MCD](http://proceedings.mlr.press/v48/gal16.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime\n",
    "\n",
    "import utils.models as models\n",
    "import utils.plotting as plotting\n",
    "import utils.dataloaders as dl\n",
    "import utils.traintest as tt\n",
    "import utils.adversarial as adv\n",
    "import utils.eval as ev\n",
    "import utils.gmm_helpers as gmm_helpers\n",
    "import model_params as params\n",
    "\n",
    "from importlib import reload\n",
    "\n",
    "import utils.mc_dropout as mc\n",
    "\n",
    "import argparse\n",
    "\n",
    "\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "datasets = ['MNIST', 'FMNIST', 'SVHN', 'CIFAR10', 'CIFAR100']\n",
    "\n",
    "for dataset in datasets:\n",
    "    print(dataset)\n",
    "\n",
    "    saving_string = dataset + '_base'\n",
    "\n",
    "    model_params = params.params_dict[dataset](augm_flag=True)\n",
    "\n",
    "    def train(model, device, train_loader, optimizer, epoch, \n",
    "              verbose=100, noise_loader=None, epsilon=.3):\n",
    "\n",
    "        criterion = nn.NLLLoss()\n",
    "        model.train()\n",
    "\n",
    "        train_loss = 0\n",
    "        correct = 0\n",
    "\n",
    "\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            \n",
    "            output = model(data)\n",
    "\n",
    "            loss = criterion(output, target)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            train_loss += loss.item()\n",
    "            _, predicted = output.max(1)\n",
    "            correct += predicted.eq(target).sum().item()\n",
    "            if (batch_idx % verbose == 0) and verbose>0:\n",
    "                print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                    epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                    100. * batch_idx / len(train_loader), loss.item()))\n",
    "        return train_loss/len(train_loader.dataset), correct/len(train_loader.dataset)\n",
    "\n",
    "\n",
    "    if dataset=='MNIST':\n",
    "        model = mc.LeNet()\n",
    "    elif dataset=='FMNIST':\n",
    "        model = mc.vgg13(in_channels=1, num_classes=10)\n",
    "    elif dataset in ['SVHN', 'CIFAR10']:\n",
    "        model = mc.vgg13(in_channels=3, num_classes=10)\n",
    "    elif dataset in ['CIFAR100']:\n",
    "        model = mc.vgg13(in_channels=3, num_classes=100)\n",
    "\n",
    "    model = model.to(device)\n",
    "\n",
    "    param_groups = [{'params':model.parameters(), 'lr':2e-4, 'weight_decay':0.}]\n",
    "\n",
    "    optimizer = optim.Adam(param_groups)\n",
    "\n",
    "\n",
    "    for epoch in range(100):\n",
    "        if epoch+1 in [50,75,90]:\n",
    "            for group in optimizer.param_groups:\n",
    "                group['lr'] *= .1\n",
    "\n",
    "        trainloss, correct_train = train(model, device, model_params.train_loader,  \n",
    "                                         optimizer, epoch, verbose=-1)\n",
    "        #print(str(epoch) + ': \\t' + str(correct_train))\n",
    "\n",
    "    model = model.to('cpu')\n",
    "\n",
    "    mc_model = mc.MC_Model(model, iterations=10, classes=model_params.classes)\n",
    "\n",
    "    torch.save(mc_model, 'SavedModels/other/mcdo/' + dataset + '.pth')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}