Python_Pandas_Athena_Examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Python_Pandas_Athena_Examples\n",
    "\n",
    "This notebook shows how to:\n",
    "\n",
    "- Read from Athena to a Pandas dataframe.\n",
    "- Write from Pandas to an Athena table.\n",
    "\n",
    "This demonstrates basic types - float64, int64 and object. More type mappings can be added if needed. See Athena Supported Types here: https://docs.aws.amazon.com/athena/latest/ug/data-types.html\n",
    "\n",
    "The user experience is kept as simple as possible:\n",
    "\n",
    "- To Read a Pandas Dataframe from Athena:\n",
    "\n",
    "```\n",
    "df=read_from_athena(<sql_statement>)\n",
    "```\n",
    "\n",
    "- To write a Pandas Dataframe to an Athena table:\n",
    "\n",
    "```\n",
    "save_to_athena(<dataframe>, <database_name>, <table_name>)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "region='us-west-2'\n",
    "defaultdb=\"default\"\n",
    "default_output=\"s3://aws-athena-query-results-959874710265-us-west-2/\"\n",
    "default_write_location=\"s3://neilawstmp2/my_home/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-08T09:51:47.840453Z",
     "start_time": "2019-11-08T09:51:47.834378Z"
    }
   },
   "outputs": [],
   "source": [
    "import boto3,time\n",
    "import pandas as pd\n",
    "\n",
    "## execute Athena SQL\n",
    "def executeQuery(query, database=defaultdb, s3_output=default_output, poll=10):\n",
    "    athena = boto3.client('athena',region_name=region)\n",
    "    response = athena.start_query_execution(\n",
    "        QueryString=query,\n",
    "        QueryExecutionContext={\n",
    "            'Database': database\n",
    "            },\n",
    "        ResultConfiguration={\n",
    "            'OutputLocation': s3_output,\n",
    "            }\n",
    "        )\n",
    "\n",
    "    print('Execution ID: ' + response['QueryExecutionId'])\n",
    "    queryExecutionId=response['QueryExecutionId']\n",
    "    state='QUEUED'\n",
    "    while( state=='RUNNING' or state=='QUEUED'):\n",
    "       response = athena.get_query_execution(QueryExecutionId=queryExecutionId)\n",
    "       state=response['QueryExecution']['Status']['State']\n",
    "       print (state)\n",
    "       if  state=='RUNNING' or state=='QUEUED':\n",
    "            time.sleep(poll)\n",
    "       elif (state=='FAILED'):\n",
    "             print (response['QueryExecution']['Status']['StateChangeReason'])\n",
    "        \n",
    "        \n",
    "    #print (response)    \n",
    "    return response\n",
    "\n",
    "## Read from Athena to a Pandas Dataframe\n",
    "def read_from_athena(sql):\n",
    "    response=executeQuery(sql)\n",
    "    return pd.read_csv(response['QueryExecution']['ResultConfiguration']['OutputLocation'])\n",
    "\n",
    "## Save Pandas Dataframe to Athena table\n",
    "def save_to_athena(df, database, tablename):\n",
    "    pandas_to_athena_types_lookup={ \"int64\":\"bigint\", \"object\":\"string\", \"float64\":\"double\"}\n",
    "    \n",
    "    ## save the data\n",
    "    table_location=default_write_location+tablename\n",
    "    file_location=table_location+'/'+tablename+\".pq\"\n",
    "    df.to_parquet(file_location)\n",
    "    \n",
    "    ## add the table to Athena\n",
    "    data_types=df.dtypes\n",
    "    columns = ''\n",
    "    for i,k in enumerate(df.columns): \n",
    "        key = str(data_types[i])\n",
    "        #print (i,key)\n",
    "        if key in pandas_to_athena_types_lookup.keys():\n",
    "            columns += (k + \" \" + pandas_to_athena_types_lookup.get(key)+\", \")\n",
    "        else:\n",
    "            raise ValueError('Type mapping does not exist for type : '+key) \n",
    "    columns=columns[:-2]\n",
    "    \n",
    "    sql = F\"CREATE EXTERNAL TABLE {database}.{tablename} \\\n",
    "    ( {columns} )  \\\n",
    "    ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'  \\\n",
    "    STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'  \\\n",
    "    OUTPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'  \\\n",
    "    LOCATION '{table_location}'  \\\n",
    "    TBLPROPERTIES ( 'classification'='parquet','typeOfData'='file')\"\n",
    "    \n",
    "    #print (sql)\n",
    "    \n",
    "    response=executeQuery(sql)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data from Athena"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-08T09:52:24.122650Z",
     "start_time": "2019-11-08T09:52:23.823070Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution ID: 012dbc0e-37ad-4c13-a3e7-fac20e0744b7\n",
      "QUEUED\n",
      "SUCCEEDED\n"
     ]
    }
   ],
   "source": [
    "sql=\"\"\"Select *\n",
    "from cdc.sales_order_fact\n",
    "LIMIT 10\"\"\"\n",
    "\n",
    "df=read_from_athena(sql)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discount</th>\n",
       "      <th>last_modified_timestamp</th>\n",
       "      <th>line_id</th>\n",
       "      <th>line_number</th>\n",
       "      <th>order_date</th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>quantity</th>\n",
       "      <th>ship_mode</th>\n",
       "      <th>site_id</th>\n",
       "      <th>supply_cost</th>\n",
       "      <th>tax</th>\n",
       "      <th>unit_price</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.822773</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>999</td>\n",
       "      <td>11</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>0.562963</td>\n",
       "      <td>0.651442</td>\n",
       "      <td>2</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>118.499121</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>846</td>\n",
       "      <td>74</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>29.767831</td>\n",
       "      <td>36.392407</td>\n",
       "      <td>133</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93.665862</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>984</td>\n",
       "      <td>18</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>621.371628</td>\n",
       "      <td>36.580441</td>\n",
       "      <td>149</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>241.543153</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>239</td>\n",
       "      <td>29</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>238.603943</td>\n",
       "      <td>161.031018</td>\n",
       "      <td>731</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>117.350523</td>\n",
       "      <td>2018-04-14T21:31:26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899501</td>\n",
       "      <td>832</td>\n",
       "      <td>90</td>\n",
       "      <td>STANDARD</td>\n",
       "      <td>44</td>\n",
       "      <td>219.627455</td>\n",
       "      <td>201.261957</td>\n",
       "      <td>462</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>152.034471</td>\n",
       "      <td>2018-04-14T21:31:26</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899501</td>\n",
       "      <td>818</td>\n",
       "      <td>67</td>\n",
       "      <td>STANDARD</td>\n",
       "      <td>44</td>\n",
       "      <td>168.970523</td>\n",
       "      <td>216.051699</td>\n",
       "      <td>524</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>193.128071</td>\n",
       "      <td>2018-04-14T19:58:03</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899502</td>\n",
       "      <td>288</td>\n",
       "      <td>20</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>158</td>\n",
       "      <td>217.463232</td>\n",
       "      <td>141.378774</td>\n",
       "      <td>700</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>72.114141</td>\n",
       "      <td>2018-04-14T19:58:03</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899502</td>\n",
       "      <td>523</td>\n",
       "      <td>93</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>158</td>\n",
       "      <td>33.959294</td>\n",
       "      <td>40.446740</td>\n",
       "      <td>152</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>611.641426</td>\n",
       "      <td>2018-04-14T03:45:39</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899503</td>\n",
       "      <td>630</td>\n",
       "      <td>87</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>237</td>\n",
       "      <td>74.448345</td>\n",
       "      <td>71.560414</td>\n",
       "      <td>332</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>47.199852</td>\n",
       "      <td>2018-04-14T18:18:12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899504</td>\n",
       "      <td>117</td>\n",
       "      <td>10</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>292</td>\n",
       "      <td>367.296527</td>\n",
       "      <td>1803.600129</td>\n",
       "      <td>227</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     discount last_modified_timestamp  line_id  line_number  \\\n",
       "0    2.822773     2018-04-14T14:21:04        1            1   \n",
       "1  118.499121     2018-04-14T14:21:04        2            2   \n",
       "2   93.665862     2018-04-14T14:21:04        3            3   \n",
       "3  241.543153     2018-04-14T14:21:04        4            4   \n",
       "4  117.350523     2018-04-14T21:31:26        1            1   \n",
       "5  152.034471     2018-04-14T21:31:26        2            2   \n",
       "6  193.128071     2018-04-14T19:58:03        1            1   \n",
       "7   72.114141     2018-04-14T19:58:03        2            2   \n",
       "8  611.641426     2018-04-14T03:45:39        1            1   \n",
       "9   47.199852     2018-04-14T18:18:12        1            1   \n",
       "\n",
       "            order_date    order_id  product_id  quantity ship_mode  site_id  \\\n",
       "0  2018-04-14T01:00:00  2550899500         999        11   NO-RUSH      327   \n",
       "1  2018-04-14T01:00:00  2550899500         846        74   NO-RUSH      327   \n",
       "2  2018-04-14T01:00:00  2550899500         984        18   NO-RUSH      327   \n",
       "3  2018-04-14T01:00:00  2550899500         239        29   NO-RUSH      327   \n",
       "4  2018-04-14T01:00:00  2550899501         832        90  STANDARD       44   \n",
       "5  2018-04-14T01:00:00  2550899501         818        67  STANDARD       44   \n",
       "6  2018-04-14T01:00:00  2550899502         288        20   ONE-DAY      158   \n",
       "7  2018-04-14T01:00:00  2550899502         523        93   ONE-DAY      158   \n",
       "8  2018-04-14T01:00:00  2550899503         630        87   ONE-DAY      237   \n",
       "9  2018-04-14T01:00:00  2550899504         117        10   ONE-DAY      292   \n",
       "\n",
       "   supply_cost          tax  unit_price  year  month  day  hour  \n",
       "0     0.562963     0.651442           2  2018      4   14     1  \n",
       "1    29.767831    36.392407         133  2018      4   14     1  \n",
       "2   621.371628    36.580441         149  2018      4   14     1  \n",
       "3   238.603943   161.031018         731  2018      4   14     1  \n",
       "4   219.627455   201.261957         462  2018      4   14     1  \n",
       "5   168.970523   216.051699         524  2018      4   14     1  \n",
       "6   217.463232   141.378774         700  2018      4   14     1  \n",
       "7    33.959294    40.446740         152  2018      4   14     1  \n",
       "8    74.448345    71.560414         332  2018      4   14     1  \n",
       "9   367.296527  1803.600129         227  2018      4   14     1  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write Data to an Athena table\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution ID: cb77e5d5-bd5f-45cd-9277-bcbeb1806142\n",
      "QUEUED\n",
      "SUCCEEDED\n"
     ]
    }
   ],
   "source": [
    "save_to_athena(df, defaultdb, \"example_table_new\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify the written data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution ID: 31ea88ae-c91e-4d09-8f76-e8d76adf3af3\n",
      "QUEUED\n",
      "SUCCEEDED\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discount</th>\n",
       "      <th>last_modified_timestamp</th>\n",
       "      <th>line_id</th>\n",
       "      <th>line_number</th>\n",
       "      <th>order_date</th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>quantity</th>\n",
       "      <th>ship_mode</th>\n",
       "      <th>site_id</th>\n",
       "      <th>supply_cost</th>\n",
       "      <th>tax</th>\n",
       "      <th>unit_price</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.822773</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>999</td>\n",
       "      <td>11</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>0.562963</td>\n",
       "      <td>0.651442</td>\n",
       "      <td>2</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>118.499121</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>846</td>\n",
       "      <td>74</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>29.767831</td>\n",
       "      <td>36.392407</td>\n",
       "      <td>133</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93.665862</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>984</td>\n",
       "      <td>18</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>621.371628</td>\n",
       "      <td>36.580441</td>\n",
       "      <td>149</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>241.543153</td>\n",
       "      <td>2018-04-14T14:21:04</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899500</td>\n",
       "      <td>239</td>\n",
       "      <td>29</td>\n",
       "      <td>NO-RUSH</td>\n",
       "      <td>327</td>\n",
       "      <td>238.603943</td>\n",
       "      <td>161.031018</td>\n",
       "      <td>731</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>117.350523</td>\n",
       "      <td>2018-04-14T21:31:26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899501</td>\n",
       "      <td>832</td>\n",
       "      <td>90</td>\n",
       "      <td>STANDARD</td>\n",
       "      <td>44</td>\n",
       "      <td>219.627455</td>\n",
       "      <td>201.261957</td>\n",
       "      <td>462</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>152.034471</td>\n",
       "      <td>2018-04-14T21:31:26</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899501</td>\n",
       "      <td>818</td>\n",
       "      <td>67</td>\n",
       "      <td>STANDARD</td>\n",
       "      <td>44</td>\n",
       "      <td>168.970523</td>\n",
       "      <td>216.051699</td>\n",
       "      <td>524</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>193.128071</td>\n",
       "      <td>2018-04-14T19:58:03</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899502</td>\n",
       "      <td>288</td>\n",
       "      <td>20</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>158</td>\n",
       "      <td>217.463232</td>\n",
       "      <td>141.378774</td>\n",
       "      <td>700</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>72.114141</td>\n",
       "      <td>2018-04-14T19:58:03</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899502</td>\n",
       "      <td>523</td>\n",
       "      <td>93</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>158</td>\n",
       "      <td>33.959294</td>\n",
       "      <td>40.446740</td>\n",
       "      <td>152</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>611.641426</td>\n",
       "      <td>2018-04-14T03:45:39</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899503</td>\n",
       "      <td>630</td>\n",
       "      <td>87</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>237</td>\n",
       "      <td>74.448345</td>\n",
       "      <td>71.560414</td>\n",
       "      <td>332</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>47.199852</td>\n",
       "      <td>2018-04-14T18:18:12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-04-14T01:00:00</td>\n",
       "      <td>2550899504</td>\n",
       "      <td>117</td>\n",
       "      <td>10</td>\n",
       "      <td>ONE-DAY</td>\n",
       "      <td>292</td>\n",
       "      <td>367.296527</td>\n",
       "      <td>1803.600129</td>\n",
       "      <td>227</td>\n",
       "      <td>2018</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     discount last_modified_timestamp  line_id  line_number  \\\n",
       "0    2.822773     2018-04-14T14:21:04        1            1   \n",
       "1  118.499121     2018-04-14T14:21:04        2            2   \n",
       "2   93.665862     2018-04-14T14:21:04        3            3   \n",
       "3  241.543153     2018-04-14T14:21:04        4            4   \n",
       "4  117.350523     2018-04-14T21:31:26        1            1   \n",
       "5  152.034471     2018-04-14T21:31:26        2            2   \n",
       "6  193.128071     2018-04-14T19:58:03        1            1   \n",
       "7   72.114141     2018-04-14T19:58:03        2            2   \n",
       "8  611.641426     2018-04-14T03:45:39        1            1   \n",
       "9   47.199852     2018-04-14T18:18:12        1            1   \n",
       "\n",
       "            order_date    order_id  product_id  quantity ship_mode  site_id  \\\n",
       "0  2018-04-14T01:00:00  2550899500         999        11   NO-RUSH      327   \n",
       "1  2018-04-14T01:00:00  2550899500         846        74   NO-RUSH      327   \n",
       "2  2018-04-14T01:00:00  2550899500         984        18   NO-RUSH      327   \n",
       "3  2018-04-14T01:00:00  2550899500         239        29   NO-RUSH      327   \n",
       "4  2018-04-14T01:00:00  2550899501         832        90  STANDARD       44   \n",
       "5  2018-04-14T01:00:00  2550899501         818        67  STANDARD       44   \n",
       "6  2018-04-14T01:00:00  2550899502         288        20   ONE-DAY      158   \n",
       "7  2018-04-14T01:00:00  2550899502         523        93   ONE-DAY      158   \n",
       "8  2018-04-14T01:00:00  2550899503         630        87   ONE-DAY      237   \n",
       "9  2018-04-14T01:00:00  2550899504         117        10   ONE-DAY      292   \n",
       "\n",
       "   supply_cost          tax  unit_price  year  month  day  hour  \n",
       "0     0.562963     0.651442           2  2018      4   14     1  \n",
       "1    29.767831    36.392407         133  2018      4   14     1  \n",
       "2   621.371628    36.580441         149  2018      4   14     1  \n",
       "3   238.603943   161.031018         731  2018      4   14     1  \n",
       "4   219.627455   201.261957         462  2018      4   14     1  \n",
       "5   168.970523   216.051699         524  2018      4   14     1  \n",
       "6   217.463232   141.378774         700  2018      4   14     1  \n",
       "7    33.959294    40.446740         152  2018      4   14     1  \n",
       "8    74.448345    71.560414         332  2018      4   14     1  \n",
       "9   367.296527  1803.600129         227  2018      4   14     1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sql=\"\"\"Select * from default.example_table_new\"\"\"\n",
    "\n",
    "new_df=read_from_athena(sql)\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}