notebooks/data_transformation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting dicto\n",
      "  Downloading https://files.pythonhosted.org/packages/90/c0/9083f33a4fea5409a73ac883fc7441a2b73d2de419c7b764eef94b901e9a/dicto-0.0.21.tar.gz\n",
      "Building wheels for collected packages: dicto\n",
      "  Running setup.py bdist_wheel for dicto ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/jovyan/.cache/pip/wheels/bb/15/ba/790a9ef4e2f517545f71681f0b467665821110a24c6db82cdd\n",
      "Successfully built dicto\n",
      "Installing collected packages: dicto\n",
      "Successfully installed dicto-0.0.21\n"
     ]
    }
   ],
   "source": [
    "!pip install dicto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pickle\n",
    "import dicto as do\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/data/train.p\", \"rb\") as fd:\n",
    "    train = pickle.load(fd)\n",
    "\n",
    "with open(\"/data/test.p\", \"rb\") as fd:\n",
    "    test = pickle.load(fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_function(image, label):\n",
    "\n",
    "    # This will convert to float values in [0, 1]\n",
    "    image = tf.image.convert_image_dtype(image, tf.float32)\n",
    "\n",
    "    return image, label\n",
    "\n",
    "\n",
    "def train_preprocess(image, label):\n",
    "\n",
    "    if label in [11, 12, 13, 15, 17, 18, 22, 26, 30, 35]:\n",
    "            image = tf.image.random_flip_left_right(image)\n",
    "\n",
    "    if label in [1, 5, 12, 15, 17]:\n",
    "        image = tf.image.random_flip_up_down(image)\n",
    "\n",
    "    image = tf.image.random_brightness(image, max_delta=32.0 / 255.0)\n",
    "    image = tf.image.random_saturation(image, lower=0.5, upper=1.5)\n",
    "\n",
    "    # Make sure the image is still in [0, 1]\n",
    "    image = tf.clip_by_value(image, 0.0, 1.0)\n",
    "\n",
    "    return image, label\n",
    "\n",
    "params = do.Dicto(\n",
    "    buffer_size = 34799,\n",
    "    batch_size = 16,\n",
    "    epochs = 10\n",
    ")\n",
    "\n",
    "def input_fn(images, labels, params, training):\n",
    "    \n",
    "    ds = tf.data.Dataset.from_tensor_slices((\n",
    "        images,\n",
    "        labels,\n",
    "    ))\n",
    "\n",
    "    ds = ds.map(parse_function, num_parallel_calls=4)\n",
    "    \n",
    "    if training:\n",
    "        ds = ds.map(train_preprocess, num_parallel_calls=4)\n",
    "        ds = ds.apply(tf.data.experimental.shuffle_and_repeat( \n",
    "        buffer_size = params.buffer_size,\n",
    "        count = params.epochs,\n",
    "        ))\n",
    "        \n",
    "    #ds  ds.map(lambda x, y: ({\"images\": x}, y))\n",
    "\n",
    "    # ds = ds.map(lambda x, y: (tf.cast(x, tf.float32), tf.cast(y, tf.int32)))\n",
    "        \n",
    "    ds = ds.batch(params.batch_size, drop_remainder=True)\n",
    "    ds = ds.prefetch(buffer_size=2)\n",
    "\n",
    "    \n",
    "    return ds\n",
    "\n",
    "ds = input_fn(train['features'], train['labels'], params, training=True)\n",
    "\n",
    "train_input_fn = lambda : input_fn(train['features'].astype(np.float32), train['labels'].astype(np.int32), params, training=True)\n",
    "eval_input_fn = lambda : input_fn(test['features'], test['labels'], params, training=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "iter_ds = ds.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "elem_ops = iter_ds.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[[ 0.39064872,  0.31792414,  0.27117264],\n",
       "          [ 0.94358993,  0.75658381,  0.72541612],\n",
       "          [ 1.        ,  1.        ,  1.        ],\n",
       "          ..., \n",
       "          [ 0.88380069,  0.90977377,  0.93574679],\n",
       "          [ 0.66302216,  0.69418985,  0.73574674],\n",
       "          [ 0.43388841,  0.44427761,  0.44947225]],\n",
       " \n",
       "         [[ 0.42202127,  0.38046438,  0.32332364],\n",
       "          [ 0.9592762 ,  0.87096781,  0.83460552],\n",
       "          [ 1.        ,  1.        ,  1.        ],\n",
       "          ..., \n",
       "          [ 0.87860608,  0.88380069,  0.93574679],\n",
       "          [ 0.64448249,  0.64448249,  0.68084478],\n",
       "          [ 0.36192468,  0.36711931,  0.35153547]],\n",
       " \n",
       "         [[ 0.47300166,  0.40547168,  0.34833097],\n",
       "          [ 0.96319777,  0.83852708,  0.80216479],\n",
       "          [ 1.        ,  1.        ,  1.        ],\n",
       "          ..., \n",
       "          [ 0.77165544,  0.79243386,  0.86515856],\n",
       "          [ 0.44193643,  0.48868793,  0.48868793],\n",
       "          [ 0.28359577,  0.30437422,  0.26281732]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.50045264,  0.42772806,  0.38617116],\n",
       "          [ 0.65339381,  0.58586383,  0.52352852],\n",
       "          [ 0.80633497,  0.72841579,  0.68685889],\n",
       "          ..., \n",
       "          [ 0.29525805,  0.30045265,  0.2796742 ],\n",
       "          [ 0.34751147,  0.34231687,  0.33192763],\n",
       "          [ 0.37496245,  0.35937861,  0.35937861]],\n",
       " \n",
       "         [[ 0.50829577,  0.47712809,  0.44596043],\n",
       "          [ 0.62202132,  0.59085363,  0.55968595],\n",
       "          [ 0.65731537,  0.62614769,  0.62095308],\n",
       "          ..., \n",
       "          [ 0.30055511,  0.31613892,  0.29016587],\n",
       "          [ 0.40241343,  0.39721882,  0.37644038],\n",
       "          [ 0.46908009,  0.44830164,  0.45349625]],\n",
       " \n",
       "         [[ 0.41957763,  0.44555068,  0.41957763],\n",
       "          [ 0.52143598,  0.53182518,  0.52663058],\n",
       "          [ 0.39233151,  0.43388841,  0.44947225],\n",
       "          ..., \n",
       "          [ 0.27320656,  0.30437422,  0.2835958 ],\n",
       "          [ 0.37898648,  0.39457029,  0.36859724],\n",
       "          [ 0.4874149 ,  0.4926095 ,  0.4874149 ]]],\n",
       " \n",
       " \n",
       "        [[[ 0.74035084,  0.5447278 ,  0.26018512],\n",
       "          [ 0.63839006,  0.50501072,  0.22046801],\n",
       "          [ 0.57564497,  0.49561736,  0.19329077],\n",
       "          ..., \n",
       "          [ 0.23054695,  0.1949791 ,  0.13718137],\n",
       "          [ 0.21486066,  0.19707675,  0.12594107],\n",
       "          [ 0.3442494 ,  0.4109391 ,  0.22420797]],\n",
       " \n",
       "         [[ 0.59917438,  0.46579504,  0.20792824],\n",
       "          [ 0.62270379,  0.49377045,  0.18699783],\n",
       "          [ 0.5795666 ,  0.55289072,  0.18832037],\n",
       "          ..., \n",
       "          [ 0.36388028,  0.29719058,  0.17270316],\n",
       "          [ 0.37564498,  0.31784725,  0.16223799],\n",
       "          [ 0.42557657,  0.43446851,  0.20772356]],\n",
       " \n",
       "         [[ 0.59525287,  0.5107792 ,  0.31515613],\n",
       "          [ 0.65407634,  0.50291306,  0.24060032],\n",
       "          [ 0.75995874,  0.57767355,  0.19087338],\n",
       "          ..., \n",
       "          [ 0.29903755,  0.31682146,  0.1745501 ],\n",
       "          [ 0.28544891,  0.28544891,  0.16540746],\n",
       "          [ 0.2736842 ,  0.24700832,  0.13585883]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.53250778,  0.39468238,  0.15015352],\n",
       "          [ 0.26584107,  0.22582725,  0.16358355],\n",
       "          [ 0.15211555,  0.14766958,  0.13433164],\n",
       "          ..., \n",
       "          [ 0.52074301,  0.49851319,  0.1828486 ],\n",
       "          [ 0.77172339,  0.66057396,  0.20263809],\n",
       "          [ 0.88544893,  0.71205568,  0.23633599]],\n",
       " \n",
       "         [[ 0.60701752,  0.47363821,  0.20243341],\n",
       "          [ 0.25015479,  0.19235706,  0.12566736],\n",
       "          [ 0.15995869,  0.15106672,  0.12439086],\n",
       "          ..., \n",
       "          [ 0.60309601,  0.45193273,  0.16739002],\n",
       "          [ 0.54035091,  0.52256703,  0.20690241],\n",
       "          [ 0.47760579,  0.41091606,  0.18417114]],\n",
       " \n",
       "         [[ 0.31682146,  0.2634697 ,  0.13898227],\n",
       "          [ 0.19133125,  0.18688527,  0.13353351],\n",
       "          [ 0.15551271,  0.15995869,  0.13328281],\n",
       "          ..., \n",
       "          [ 0.8148607 ,  0.57477784,  0.20131552],\n",
       "          [ 0.62270379,  0.42263478,  0.14698404],\n",
       "          [ 0.59525287,  0.44853556,  0.19511473]]],\n",
       " \n",
       " \n",
       "        [[[ 0.64220595,  0.47791845,  0.59871799],\n",
       "          [ 0.6618138 ,  0.49752629,  0.61349392],\n",
       "          [ 0.64612752,  0.47700804,  0.59297562],\n",
       "          ..., \n",
       "          [ 0.61867648,  0.464053  ,  0.57518864],\n",
       "          [ 0.60299021,  0.45319873,  0.56916636],\n",
       "          [ 0.59514707,  0.45018756,  0.56132317]],\n",
       " \n",
       "         [[ 0.62651962,  0.46706414,  0.59269577],\n",
       "          [ 0.63828439,  0.47399688,  0.59479642],\n",
       "          [ 0.65004909,  0.47126561,  0.58723319],\n",
       "          ..., \n",
       "          [ 0.62651962,  0.46223217,  0.58303177],\n",
       "          [ 0.61475492,  0.45529944,  0.58093107],\n",
       "          [ 0.60691178,  0.4571203 ,  0.5779199 ]],\n",
       " \n",
       "         [[ 0.62259805,  0.47763854,  0.60327017],\n",
       "          [ 0.64220595,  0.47791845,  0.60355008],\n",
       "          [ 0.65789223,  0.47427678,  0.59507632],\n",
       "          ..., \n",
       "          [ 0.63828439,  0.46916488,  0.58996445],\n",
       "          [ 0.62259805,  0.4583106 ,  0.58394217],\n",
       "          [ 0.61867648,  0.44955707,  0.57518864]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.63044125,  0.48548168,  0.58695334],\n",
       "          [ 0.63044125,  0.47581768,  0.58695334],\n",
       "          [ 0.63828439,  0.46433288,  0.57063651],\n",
       "          ..., \n",
       "          [ 0.58338237,  0.43359089,  0.54472655],\n",
       "          [ 0.59906864,  0.43961319,  0.55074877],\n",
       "          [ 0.60299021,  0.43387079,  0.54500639]],\n",
       " \n",
       "         [[ 0.64220595,  0.48275042,  0.57455808],\n",
       "          [ 0.63828439,  0.47882885,  0.59962839],\n",
       "          [ 0.64220595,  0.47791845,  0.58422208],\n",
       "          ..., \n",
       "          [ 0.58338237,  0.42875889,  0.55439043],\n",
       "          [ 0.5912255 ,  0.43177006,  0.55740166],\n",
       "          [ 0.59514707,  0.43085963,  0.5468272 ]],\n",
       " \n",
       "         [[ 0.63436282,  0.46041131,  0.56188297],\n",
       "          [ 0.62259805,  0.45347863,  0.56944621],\n",
       "          [ 0.63044125,  0.46615374,  0.57245737],\n",
       "          ..., \n",
       "          [ 0.59514707,  0.43085963,  0.55165923],\n",
       "          [ 0.60299021,  0.44353473,  0.56433427],\n",
       "          [ 0.59514707,  0.4356916 ,  0.55165923]]],\n",
       " \n",
       " \n",
       "        ..., \n",
       "        [[[ 0.1713649 ,  0.24618   ,  0.32930791],\n",
       "          [ 0.15201539,  0.20189215,  0.28224909],\n",
       "          [ 0.19285136,  0.23441534,  0.31754321],\n",
       "          ..., \n",
       "          [ 0.1656587 ,  0.20999357,  0.26264125],\n",
       "          [ 0.20440474,  0.24042684,  0.30970007],\n",
       "          [ 0.17972468,  0.20189214,  0.28224909]],\n",
       " \n",
       "         [[ 0.22192264,  0.27179939,  0.34107262],\n",
       "          [ 0.18040566,  0.22474054,  0.2940138 ],\n",
       "          [ 0.16147882,  0.20581369,  0.28617066],\n",
       "          ..., \n",
       "          [ 0.179983  ,  0.20215045,  0.25479811],\n",
       "          [ 0.18181464,  0.20398208,  0.27048439],\n",
       "          [ 0.17167021,  0.18275392,  0.25479811]],\n",
       " \n",
       "         [[ 0.16774863,  0.17883234,  0.25087655],\n",
       "          [ 0.1555143 ,  0.17491078,  0.24695498],\n",
       "          [ 0.18202595,  0.2041934 ,  0.29009223],\n",
       "          ..., \n",
       "          [ 0.2780692 ,  0.26698548,  0.3057785 ],\n",
       "          [ 0.2672438 ,  0.25061822,  0.27832752],\n",
       "          [ 0.22342557,  0.19848719,  0.21234185]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.11775455,  0.12052548,  0.13715106],\n",
       "          [ 0.11106205,  0.11383298,  0.13322949],\n",
       "          [ 0.10436955,  0.11268234,  0.12930793],\n",
       "          ..., \n",
       "          [ 0.09124287,  0.08847194,  0.0940138 ],\n",
       "          [ 0.0940138 ,  0.09124287,  0.09124287],\n",
       "          [ 0.10185694,  0.09631508,  0.09631508]],\n",
       " \n",
       "         [[ 0.08293008,  0.08015915,  0.0940138 ],\n",
       "          [ 0.08408072,  0.08408072,  0.09793537],\n",
       "          [ 0.09192386,  0.09192386,  0.10577851],\n",
       "          ..., \n",
       "          [ 0.19989616,  0.19712523,  0.19712523],\n",
       "          [ 0.21166086,  0.20888993,  0.206119  ],\n",
       "          [ 0.219504  ,  0.21396214,  0.21119121]],\n",
       " \n",
       "         [[ 0.18672249,  0.20057712,  0.21166086],\n",
       "          [ 0.22640783,  0.23194969,  0.24303341],\n",
       "          [ 0.23310032,  0.23864219,  0.24695498],\n",
       "          ..., \n",
       "          [ 0.25479811,  0.25479811,  0.25202718],\n",
       "          [ 0.25871968,  0.25871968,  0.25594875],\n",
       "          [ 0.25479811,  0.25479811,  0.25479811]]],\n",
       " \n",
       " \n",
       "        [[[ 0.66718042,  0.75546861,  0.92820621],\n",
       "          [ 0.66709745,  0.75538564,  0.9242847 ],\n",
       "          [ 0.66718042,  0.75930715,  0.92820621],\n",
       "          ..., \n",
       "          [ 0.68245196,  0.76306289,  0.9242847 ],\n",
       "          [ 0.67861331,  0.76306283,  0.9242847 ],\n",
       "          [ 0.67460883,  0.76289701,  0.91644156]],\n",
       " \n",
       "         [[ 0.66301   ,  0.75129813,  0.91251999],\n",
       "          [ 0.66309297,  0.74754256,  0.91644156],\n",
       "          [ 0.65917134,  0.74745941,  0.91251993],\n",
       "          ..., \n",
       "          [ 0.6785304 ,  0.76681852,  0.92036313],\n",
       "          [ 0.67085314,  0.76681852,  0.92036307],\n",
       "          [ 0.6785304 ,  0.75914133,  0.92036313]],\n",
       " \n",
       "         [[ 0.65524983,  0.73969942,  0.90859842],\n",
       "          [ 0.65524983,  0.73586076,  0.90859842],\n",
       "          [ 0.65516686,  0.73961633,  0.90467685],\n",
       "          ..., \n",
       "          [ 0.67836452,  0.76281404,  0.91252005],\n",
       "          [ 0.67460883,  0.76289701,  0.91644156],\n",
       "          [ 0.67452586,  0.75513673,  0.91251999]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.20713532,  0.21097393,  0.230167  ],\n",
       "          [ 0.2031308 ,  0.2031308 ,  0.22232386],\n",
       "          [ 0.21472959,  0.21089098,  0.22624543],\n",
       "          ..., \n",
       "          [ 0.51266313,  0.59327406,  0.70075524],\n",
       "          [ 0.32442784,  0.39736149,  0.51251996],\n",
       "          [ 0.19570243,  0.20721827,  0.23408857]],\n",
       " \n",
       "         [[ 0.20713532,  0.21097393,  0.230167  ],\n",
       "          [ 0.19528766,  0.19528766,  0.21448073],\n",
       "          [ 0.19904332,  0.1952047 ,  0.21055916],\n",
       "          ..., \n",
       "          [ 0.48896784,  0.56574011,  0.66938269],\n",
       "          [ 0.32409602,  0.39319104,  0.49683368],\n",
       "          [ 0.19970696,  0.21506141,  0.24193171]],\n",
       " \n",
       "         [[ 0.1952047 ,  0.1952047 ,  0.21055916],\n",
       "          [ 0.19128314,  0.19128314,  0.20663759],\n",
       "          [ 0.19128314,  0.18744452,  0.20663759],\n",
       "          ..., \n",
       "          [ 0.46502364,  0.52260286,  0.62624544],\n",
       "          [ 0.31550631,  0.3692469 ,  0.45369643],\n",
       "          [ 0.199624  ,  0.21497846,  0.23801014]]],\n",
       " \n",
       " \n",
       "        [[[ 0.21312609,  0.1850045 ,  0.16250724],\n",
       "          [ 0.25626338,  0.22814175,  0.18314719],\n",
       "          [ 0.28371435,  0.26121709,  0.19372523],\n",
       "          ..., \n",
       "          [ 0.3895967 ,  0.32772923,  0.27148604],\n",
       "          [ 0.34253788,  0.33128923,  0.26379743],\n",
       "          [ 0.28763592,  0.27076298,  0.20327112]],\n",
       " \n",
       "         [[ 0.23273395,  0.1933637 ,  0.1596178 ],\n",
       "          [ 0.3150869 ,  0.25321943,  0.18010324],\n",
       "          [ 0.36606729,  0.29295117,  0.20296207],\n",
       "          ..., \n",
       "          [ 0.44449866,  0.36575818,  0.32076365],\n",
       "          [ 0.43665552,  0.41415825,  0.35791507],\n",
       "          [ 0.43273395,  0.41586101,  0.32024759]],\n",
       " \n",
       "         [[ 0.24842022,  0.20342566,  0.1528068 ],\n",
       "          [ 0.35430259,  0.28118646,  0.1799487 ],\n",
       "          [ 0.41704768,  0.3383072 ,  0.19769928],\n",
       "          ..., \n",
       "          [ 0.70332217,  0.57958716,  0.54584128],\n",
       "          [ 0.74645942,  0.71271354,  0.62834877],\n",
       "          [ 0.78567511,  0.77442646,  0.56632674]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.23273395,  0.17649075,  0.13149621],\n",
       "          [ 0.25626338,  0.18877152,  0.1212797 ],\n",
       "          [ 0.28763592,  0.20889546,  0.14702794],\n",
       "          ..., \n",
       "          [ 0.23273396,  0.14836916,  0.1258719 ],\n",
       "          [ 0.35822415,  0.15012436,  0.08825687],\n",
       "          [ 0.49940062,  0.38128996,  0.24630633]],\n",
       " \n",
       "         [[ 0.2405771 ,  0.18433389,  0.13371503],\n",
       "          [ 0.24842022,  0.19780134,  0.13030952],\n",
       "          [ 0.25626338,  0.20564446,  0.14377697],\n",
       "          ..., \n",
       "          [ 0.18567511,  0.1519292 ,  0.14068057],\n",
       "          [ 0.21312609,  0.16250724,  0.12876132],\n",
       "          [ 0.28371435,  0.24996845,  0.17122796]],\n",
       " \n",
       "         [[ 0.28371435,  0.18810092,  0.14310637],\n",
       "          [ 0.3033222 ,  0.23020606,  0.1627142 ],\n",
       "          [ 0.31116533,  0.24929783,  0.19305462],\n",
       "          ..., \n",
       "          [ 0.16606727,  0.14919433,  0.14357001],\n",
       "          [ 0.1621457 ,  0.15652138,  0.12277548],\n",
       "          [ 0.27194965,  0.18196052,  0.10884438]]]], dtype=float32),\n",
       " array([11, 35, 18,  6,  9, 26, 24, 35, 15,  7,  2, 41, 35,  5, 30, 38], dtype=uint8))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(elem_ops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}