diff --git a/.github/workflows/cloud_functions/test-local.sh b/.github/workflows/cloud_functions/test-local.sh index 74c35ff2..491ea773 100755 --- a/.github/workflows/cloud_functions/test-local.sh +++ b/.github/workflows/cloud_functions/test-local.sh @@ -2,4 +2,4 @@ flake8 usl_pipeline/cloud_functions --show-source --statistics black usl_pipeline/cloud_functions --check pytest usl_pipeline/cloud_functions -mypy usl_pipeline/cloud_functions +mypy usl_pipeline/cloud_functions \ No newline at end of file diff --git a/.github/workflows/usl_models/test-local.sh b/.github/workflows/usl_models/test-local.sh new file mode 100755 index 00000000..89d2cb4e --- /dev/null +++ b/.github/workflows/usl_models/test-local.sh @@ -0,0 +1,6 @@ +# .github/workflows/usl_models/test-local.sh +# if it fails, we should give permission: chmod +x /home/elhajjas/climateiq-cnn/.github/workflows/usl_models/test-local.sh +flake8 usl_models --show-source --statistics +black usl_models --check +pytest usl_models -k "not integration" +mypy usl_models \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4161b45c..b17f0c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ build dist __pycache__ +.mypy_cache +.pytest_cache *.egg-info venv diff --git a/.vscode/settings.json b/.vscode/settings.json index 64a2683b..58888b52 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,9 @@ { "jupyter.notebookFileRoot": "${fileDirname}/..", "python.testing.pytestArgs": [ - "usl_models", "--rootdir=usl_models", "-k", "not integration" + "usl_models" ], - "python.testing.cwd": "usl_models", "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "python.testing.pytestPath": "pytest" } \ No newline at end of file diff --git a/usl_models/notebooks/train_atmo_model.ipynb b/usl_models/notebooks/train_atmo_model.ipynb new file mode 100644 index 00000000..6d61cb74 --- /dev/null +++ b/usl_models/notebooks/train_atmo_model.ipynb @@ -0,0 +1,1585 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Atmo Model Training Notebook\n", + "\n", + "Train an Atmo Model using `usl_models` lib." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n", + "['NYC_Heat_Test/NYC_summer_2000_01p', 'NYC_Heat_Test/NYC_summer_2010_99p', 'NYC_Heat_Test/NYC_summer_2015_50p', 'NYC_Heat_Test/NYC_summer_2017_25p', 'NYC_Heat_Test/NYC_summer_2018_75p', 'PHX_Heat_Test/PHX_summer_2008_25p', 'PHX_Heat_Test/PHX_summer_2009_50p', 'PHX_Heat_Test/PHX_summer_2011_99p', 'PHX_Heat_Test/PHX_summer_2015_75p', 'PHX_Heat_Test/PHX_summer_2020_01p']\n" + ] + } + ], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "import tensorflow as tf\n", + "import keras\n", + "from usl_models.atmo_ml.model import AtmoModel, AtmoModelParams\n", + "from usl_models.atmo_ml import dataset\n", + "from google.cloud import storage\n", + "\n", + "import logging\n", + "\n", + "logging.getLogger().setLevel(logging.INFO)\n", + "\n", + "# climateiq-study-area-feature-chunks/NYC_Heat/NYC_summer_2000_01p\n", + "# Define bucket names and folder paths\n", + "data_bucket_name = \"climateiq-study-area-feature-chunks\"\n", + "label_bucket_name = \"climateiq-study-area-label-chunks\"\n", + "time_steps_per_day = 6\n", + "batch_size = 50\n", + "\n", + "sim_dirs = [\n", + " ('NYC_Heat_Test', [\n", + " 'NYC_summer_2000_01p',\n", + " # 'NYC_summer_2010_99p',\n", + " # 'NYC_summer_2015_50p',\n", + " # 'NYC_summer_2017_25p',\n", + " # 'NYC_summer_2018_75p'\n", + " ]),\n", + " ('PHX_Heat_Test', [\n", + " 'PHX_summer_2008_25p',\n", + " # 'PHX_summer_2009_50p',\n", + " # 'PHX_summer_2011_99p',\n", + " # 'PHX_summer_2015_75p',\n", + " # 'PHX_summer_2020_01p'\n", + " ])\n", + "]\n", + "\n", + "sim_names = []\n", + "for sim_dir, subdirs in sim_dirs:\n", + " for subdir in subdirs:\n", + " sim_names.append(sim_dir + '/' + subdir)\n", + "\n", + "print(sim_names)\n", + "client = storage.Client(project=\"climateiq\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:sim_name_dates [('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-14'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-08'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-04'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-20'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-23'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-09'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-12'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-13'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-19'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-23'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-28'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-24'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-23'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-21'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-31'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-06'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-25'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-08'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-23'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-26'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-22'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-17'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-21'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-10'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-25'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-31'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-18'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-29'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-05'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-24'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-09'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-04'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-20'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-07'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-24'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-18'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-11'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-25'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-10'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-30'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-22'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-08'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-15'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-21'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-09'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-27'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-03'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-02'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-13'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-17'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-02'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-11'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-23'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-22'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-04'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-14'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-20'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-15'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-11'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-04'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-20'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-22'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-23'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-17'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-17'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-13'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-07'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-20'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-03'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-29'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-21'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-08'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-20'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-01'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-31'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-28'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-03'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-23'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-04'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-08'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-11'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-29'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-17'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-31'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-09'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-07'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-05'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-15'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-11'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-27'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-22'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-26'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-30'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-14'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-14'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-11'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-18'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-03'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-15'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-19'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-04'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-03'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-07'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-15'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-07'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-09'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-14'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-17'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-17'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-16'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-07'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-27'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-17'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-06'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-18'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-28'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-09'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-03'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-16'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-14'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-31'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-17'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-05'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-09'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-16'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-03'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-05'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-01'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-26'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-06'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-13'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-31'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-06'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-18'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-11'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-20'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-12'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-10'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-19'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-01'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-21'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-14'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-27'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-23'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-12'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-05'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-09'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-14'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-21'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-08'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-10'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-26'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-09'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-08'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-17'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-23'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-19'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-19'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-21'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-27'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-29'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-17'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-26'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-11'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-22'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-05'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-07'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-29'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-21'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-29'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-05'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-02'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-12'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-05'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-18'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-22'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-23'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-03'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-14'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-20'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-04'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-31'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-21'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-16'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-19'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-25'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-08'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-22'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-18'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-22'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-01'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-21'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-27'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-05'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-06'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-14'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-05'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-23'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-13'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-26'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-29'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-04'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-31'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-12'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-18'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-13'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-17'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-22'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-14'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-15'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-18'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-09'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-09'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-03'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-07'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-02'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-30'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-27'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-14'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-06'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-14'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-05'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-28'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-20'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-11'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-17'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-08'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-16'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-23'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-11'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-24'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-11'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-20'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-28'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-15'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-22'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-25'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-13'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-26'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-03'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-23'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-27'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-03'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-21'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-31'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-05'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-27'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-01'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-16'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-19'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-24'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-07'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-12'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-06'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-09'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-02'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-01'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-11'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-31'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-26'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-27'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-31'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-14'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-05'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-28'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-27'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-14'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-10'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-16'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-25'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-07'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-03'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-17'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-20'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-20'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-08'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-31'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-15'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-17'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-24'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-12'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-21'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-14'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-15'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-08'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-23'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-19'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-24'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-22'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-31'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-03'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-11'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-02'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-20'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-09'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-19'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-27'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-08'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-16'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-08'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-01'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-19'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-25'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-14'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-14'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-04'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-06'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-01'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-03'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-14'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-06'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-01'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-20'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-16'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-12'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-18'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-01'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-13'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-06'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-08'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-22'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-11'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-26'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-31'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-17'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-10'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-15'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-20'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-21'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-14'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-06'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-25'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-03'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-19'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-14'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-23'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-22'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-02'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-31'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-08'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-20'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-01'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-12'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-04'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-15'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-07'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-14'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-11'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-14'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-04'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-20'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-24'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-23'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-02'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-08'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-07'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-23'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-03'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-13'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-18'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-31'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-11'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-15'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-14'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-11'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-02'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-06'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-31'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-30'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-20'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-18'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-17'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-06'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-17'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-29'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-05'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-06'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-21'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-21'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-11'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-12'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-18'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-24'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-07'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-27'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-15'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-31'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-22'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-06'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-16'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-09'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-13'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-23'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-07'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-08')]\n", + "INFO:root:sim_name_dates [('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-18'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-23'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-18'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-08'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-20'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-15'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-23'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-06'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-09'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-08'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-31'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-09'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-27'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-03'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-12'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-24'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-07'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-25'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-21'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-20'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-06'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-30'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-13'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-05'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-25'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-07'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-25'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-16'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-22'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-03'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-21'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-11'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-24'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-24'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-30'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-07'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-04'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-14'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-28'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-15'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-03'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-12'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-25'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-26'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-12'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-15'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-14'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-03'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-04'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-30'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-01'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-16'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-06'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-20'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-19'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-17'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-20'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-08'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-21'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-11'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-21'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-13'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-03'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-31'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-29'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-14'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-12'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-08'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-31'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-06'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-20'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-18'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-20'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-28'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-03'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-21'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-29'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-16'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-04'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-14'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-15'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-11'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-11'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-09'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-20'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-24'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-13'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-09'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-24'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-03'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-11'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-27'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-08'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-31'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-26'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-03'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-11'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-27'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-10'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-18'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-24'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-30'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-16'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-09'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-11'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-16'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-09'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-13'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-06'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-26'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-03'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-02'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-01'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-25'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-20'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-22'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-17'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-11'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-06'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-17'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-09'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-14'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-02'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-28'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-21'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-14'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-21'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-16'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-29'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-07'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-14'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-17'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-02'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-22'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-11'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-12'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-11'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-22'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-21'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-08'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-06'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-14'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-11'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-14'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-24'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-07'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-28'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-11'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-08'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-24'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-25'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-23'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-19'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-07'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-23'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-17'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-30'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-05'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-01'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-20'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-31'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-29'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-17'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-09'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-20'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-21'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-19'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-03'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-23'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-20'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-15'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-29'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-29'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-31'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-25'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-09'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-10'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-30'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-05'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-20'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-04'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-24'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-10'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-30'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-08'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-19'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-22'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-22'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-19'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-03'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-06'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-05'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-11'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-02'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-21'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-08'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-01'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-01'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-13'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-24'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-15'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-08'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-07'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-10'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-01'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-24'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-12'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-11'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-02'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-13'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-24'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-26'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-09'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-08'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-20'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-27'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-20'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-06'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-26'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-22'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-21'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-05'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-31'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-09'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-19'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-20'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-12'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-20'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-22'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-23'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-26'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-25'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-31'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-19'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-15'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-29'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-09'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-11'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-18'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-31'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-22'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-17'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-14'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-01'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-03'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-10'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-30'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-07'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-26'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-20'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-06'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-23'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-09'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-09'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-23'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-29'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-23'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-27'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-08'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-01'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-24'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-16'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-24'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-23'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-19'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-07'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-26'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-01'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-14'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-31'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-25'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-12'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-16'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-27'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-31'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-24'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-18'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-12'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-27'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-02'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-07'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-11'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-05'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-30'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-08'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-26'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-07'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-27'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-21'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-06'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-20'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-07'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-28'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-23'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-05'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-10'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-05-29'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-15'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-08'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-12'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-18'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-10'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-25'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-22'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-09'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-09'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-24'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-11'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-17'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-10'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-14'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-28'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-10'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-07'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-01'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-05'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-15'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-19'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-17'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-12'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-05-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-12'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-23'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-21'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-05'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-13'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-06'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-27'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-16'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-08'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-02'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-23'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-01'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-22'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-30'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-20'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-26'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-04'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-24'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-28'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-30'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-01'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-06'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-06'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-15'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-13'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-24'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-04'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-01'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-16'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-04'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-13'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-26'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-19'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-17'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-04'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-13'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-09'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-04'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-21'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-31'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-18'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-08'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-17'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-19'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-29'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-30'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-16'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-26'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-02'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-29'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-17'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-09'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-05'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-12'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-21'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-11'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-16'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-25'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-31'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-08'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-17'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-14'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-23'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-20'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-23'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-21'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-22'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-22'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-10'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-05-31'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-16'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-10'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-21'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-11'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-14'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-03'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-17'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-30'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-16'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-29'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-01'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-26'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-31'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-22'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-13'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-15'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-07'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-19'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-20'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-17'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-21'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-01'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-17'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-18'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-11'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-29'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-16'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-05-27'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-23'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-19'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-14'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-13'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-15'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-07'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-25'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-31'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-28'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-29'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-05'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-06'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-28'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-30'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-26'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-20'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-27'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-03'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-10'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-04'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-02'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-03'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-21'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-07'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-21'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-11'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-20'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-28'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-27'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-04'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-06-24'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-04'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-30'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-20'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-09'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-29'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-14'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-11'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-02'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-03'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-07-22'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-15'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-10'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-09'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-09'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-31'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-17'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-18'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-18'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-15'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-28'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-13'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-28'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-15'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-05'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-26'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-05'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-08'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-27'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-03'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-01'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-21'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-30'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-07-04'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-13'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-25'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-08-15'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-09'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-16'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-22'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-06-21'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-05-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-31'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-22'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-12'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-20'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-08'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-27'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-08-11'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-11'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-05'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-17'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-11'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-19'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-18'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-31'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-07-23'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-05-31'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-08-19'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-07-24'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-10'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-08-22'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-07-26'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-16'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-03'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-14'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-28'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-30'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-08'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-14'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-14'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-25'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-06-04'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-12'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-07'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-12'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-06-25'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-29'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-02'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-29'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-05-26'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-05-29'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-08-05'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-08-22'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-19'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-07-17'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-08-01'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-06-07'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-07-23'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-18'), ('PHX_Heat_Test/PHX_summer_2020_01p', '2020-08-20'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-02'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-06-10'), ('NYC_Heat_Test/NYC_summer_2017_25p', '2017-07-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-06'), ('NYC_Heat_Test/NYC_summer_2018_75p', '2018-08-14'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-15'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-07-13'), ('PHX_Heat_Test/PHX_summer_2015_75p', '2015-08-22'), ('PHX_Heat_Test/PHX_summer_2009_50p', '2009-05-30'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-05-25'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-02'), ('NYC_Heat_Test/NYC_summer_2000_01p', '2000-06-08'), ('NYC_Heat_Test/NYC_summer_2010_99p', '2010-06-30'), ('PHX_Heat_Test/PHX_summer_2008_25p', '2008-06-27'), ('NYC_Heat_Test/NYC_summer_2015_50p', '2015-07-12'), ('PHX_Heat_Test/PHX_summer_2011_99p', '2011-06-23')]\n" + ] + } + ], + "source": [ + "# Create the training dataset using create_atmo_dataset with sim_names\n", + "train_frac = 0.8\n", + "\n", + "train_ds = dataset.load_dataset(\n", + " data_bucket_name=data_bucket_name,\n", + " label_bucket_name=label_bucket_name,\n", + " sim_names=sim_names,\n", + " hash_range=(0.0, train_frac),\n", + ").batch(batch_size=batch_size)\n", + "\n", + "val_ds = dataset.load_dataset(\n", + " data_bucket_name=data_bucket_name,\n", + " label_bucket_name=label_bucket_name,\n", + " sim_names=sim_names,\n", + " hash_range=(train_frac, 1.0),\n", + ").batch(batch_size=batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2008-07-25, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2018-07-20, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2011-06-07, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-07-15, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2018-08-23, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-28, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2009-06-25, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-25 PHX_Heat_Test/PHX_summer_2009_50p #1\n", + "INFO:root:load_day (2017-07-24, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-31, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-06-30, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2008-08-23, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-07-06, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-24, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-08-24 PHX_Heat_Test/PHX_summer_2011_99p #2\n", + "INFO:root:load_day (2008-06-14, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-06-07, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-07-25, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-25 PHX_Heat_Test/PHX_summer_2011_99p #3\n", + "INFO:root:load_day (2015-07-12, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-06-30, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-15, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-06-07, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2020-06-07, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-25, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-07-17, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-06-23, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-15, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-08-08, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-20, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-06-17, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2020-06-11, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-12, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-08-24, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-04, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-07-03, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2011-06-26, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-26 PHX_Heat_Test/PHX_summer_2011_99p #4\n", + "INFO:root:load_day (2020-07-21, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-06-23, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-07-05, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2008-07-28, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-02, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-02 PHX_Heat_Test/PHX_summer_2009_50p #5\n", + "INFO:root:load_day (2015-07-31, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-07-15, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-15 PHX_Heat_Test/PHX_summer_2011_99p #6\n", + "INFO:root:load_day (2000-08-18, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-18 NYC_Heat_Test/NYC_summer_2000_01p #7\n", + "INFO:root:load_day (2010-07-27, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-05-31, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-04, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-07-03, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-08-17, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-06-12, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-07-18, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-08-28, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-28 NYC_Heat_Test/NYC_summer_2015_50p #8\n", + "INFO:root:load_day (2010-05-31, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-08-23, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2017-08-23 NYC_Heat_Test/NYC_summer_2017_25p #9\n", + "INFO:root:load_day (2011-06-27, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-08-16, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-16 NYC_Heat_Test/NYC_summer_2000_01p #10\n", + "INFO:root:load_day (2008-08-06, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-07-18, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-08-12, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-12 PHX_Heat_Test/PHX_summer_2009_50p #11\n", + "INFO:root:load_day (2015-08-01, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-08-07, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-23, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-09, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-07-08, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-05-27, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-30, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-08-19, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-07-26, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-06-05, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-05-29, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-06-08, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-07-30, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-08-22, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-06-15, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-07-14, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-19, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-07-31, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-07-08, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2020-08-01, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-01 PHX_Heat_Test/PHX_summer_2020_01p #12\n", + "INFO:root:load_day (2015-07-25, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-18, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-08-22, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2017-05-26, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-08-25, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2017-08-25 NYC_Heat_Test/NYC_summer_2017_25p #13\n", + "INFO:root:load_day (2017-08-26, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-18, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-17, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2020-06-15, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2018-06-24, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2009-06-12, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2000-07-28, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-28 NYC_Heat_Test/NYC_summer_2000_01p #14\n", + "INFO:root:load_day (2018-07-14, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-06-12, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2000-07-23, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2000-08-24, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-24 NYC_Heat_Test/NYC_summer_2000_01p #15\n", + "INFO:root:load_day (2020-08-22, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-22 PHX_Heat_Test/PHX_summer_2020_01p #16\n", + "INFO:root:load_day (2017-07-10, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-26, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-08-04, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-21, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-06-28, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-07-16, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-07-10, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-08-22, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-22 NYC_Heat_Test/NYC_summer_2000_01p #17\n", + "INFO:root:load_day (2015-05-31, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2009-08-15, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-15 PHX_Heat_Test/PHX_summer_2009_50p #18\n", + "INFO:root:load_day (2009-07-16, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-16 PHX_Heat_Test/PHX_summer_2009_50p #19\n", + "INFO:root:load_day (2015-05-29, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-08-12, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-08-28, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-28 NYC_Heat_Test/NYC_summer_2000_01p #20\n", + "INFO:root:load_day (2011-06-25, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-25 PHX_Heat_Test/PHX_summer_2011_99p #21\n", + "INFO:root:load_day (2020-08-31, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2011-07-10, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-08-14, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-14 NYC_Heat_Test/NYC_summer_2000_01p #22\n", + "INFO:root:load_day (2009-08-10, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-10 PHX_Heat_Test/PHX_summer_2009_50p #23\n", + "INFO:root:load_day (2015-08-16, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-06-19, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-07, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-07-16, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-08-30, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-05-24, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2008-06-30, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-01, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-06-11, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-06-17, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2000-07-24, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-07-24, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-31, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-08, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-06-28, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-28 PHX_Heat_Test/PHX_summer_2009_50p #24\n", + "INFO:root:load_day (2020-05-24, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-07-15, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-06, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-08-22, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2017-08-22 NYC_Heat_Test/NYC_summer_2017_25p #25\n", + "INFO:root:load_day (2017-06-19, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-05-25, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2000-07-22, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-22 NYC_Heat_Test/NYC_summer_2000_01p #26\n", + "INFO:root:load_day (2000-08-02, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-02 NYC_Heat_Test/NYC_summer_2000_01p #27\n", + "INFO:root:load_day (2011-08-11, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-08-09, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-09 NYC_Heat_Test/NYC_summer_2000_01p #28\n", + "INFO:root:load_day (2000-08-19, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-19 NYC_Heat_Test/NYC_summer_2000_01p #29\n", + "INFO:root:load_day (2000-07-17, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-17 NYC_Heat_Test/NYC_summer_2000_01p #30\n", + "INFO:root:load_day (2020-08-30, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-08-09, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-07-21, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2018-08-25, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2010-06-06, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-07-16, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-07-09, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-05-25, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-06-20, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-06-13, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-05-28, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-07-22, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-08-13, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-26 PHX_Heat_Test/PHX_summer_2009_50p #31\n", + "INFO:root:load_day (2000-08-01, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-01 NYC_Heat_Test/NYC_summer_2000_01p #32\n", + "INFO:root:load_day (2010-06-18, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-07-10, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-07-08, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2009-08-14, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-14 PHX_Heat_Test/PHX_summer_2009_50p #33\n", + "INFO:root:load_day (2015-06-25, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-16, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-06-19, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-08-14, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2010-07-24, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-08-01, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-26, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-06-29, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-08-15, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-07, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-21, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2018-08-14, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2010-07-21, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-11, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2010-08-05, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-07-23, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-08-13, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2011-06-10, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-06-15, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-07-07, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2009-07-30, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-30 PHX_Heat_Test/PHX_summer_2009_50p #34\n", + "INFO:root:load_day (2015-07-31, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2018-05-30, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2020-06-20, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2011-07-28, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-08-18, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-18 NYC_Heat_Test/NYC_summer_2015_50p #35\n", + "INFO:root:load_day (2020-07-05, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-07-10, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-06-18, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-06-04, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-08-31, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-08-22, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-08-23, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-23 PHX_Heat_Test/PHX_summer_2020_01p #36\n", + "INFO:root:load_day (2018-07-28, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-05-28, NYC_Heat_Test/NYC_summer_2018_75p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of samples: 158\n" + ] + } + ], + "source": [ + "num_samples = 0\n", + "for batch in train_ds:\n", + " num_samples += batch[0]['spatiotemporal'].shape[0]\n", + "print(\"Number of samples:\", num_samples)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2015-06-18, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2020-08-09, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-09, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-02, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-05-24, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-08-30, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-30 PHX_Heat_Test/PHX_summer_2009_50p #1\n", + "INFO:root:load_day (2011-06-24, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-05-31, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-08-08, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-08 PHX_Heat_Test/PHX_summer_2009_50p #2\n", + "INFO:root:load_day (2008-08-19, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-07-27, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-27 PHX_Heat_Test/PHX_summer_2009_50p #3\n", + "INFO:root:load_day (2015-07-01, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-14, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-07-23, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2010-08-28, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-07-06, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-06 PHX_Heat_Test/PHX_summer_2009_50p #4\n", + "INFO:root:load_day (2000-06-11, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-08-09, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-08-08, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-08 NYC_Heat_Test/NYC_summer_2000_01p #5\n", + "INFO:root:load_day (2000-07-25, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2018-06-22, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-13, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-07-31, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-31 PHX_Heat_Test/PHX_summer_2009_50p #6\n", + "INFO:root:load_day (2010-07-23, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-07-31, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2009-08-09, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-09 PHX_Heat_Test/PHX_summer_2009_50p #7\n", + "INFO:root:load_day (2020-08-27, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-27 PHX_Heat_Test/PHX_summer_2020_01p #8\n", + "INFO:root:load_day (2009-08-03, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-03 PHX_Heat_Test/PHX_summer_2009_50p #9\n", + "INFO:root:load_day (2000-06-05, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-06-16, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-12, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2020-05-26, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-06-02, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2009-05-24, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2011-07-07, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-08-21, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-21 PHX_Heat_Test/PHX_summer_2009_50p #10\n", + "INFO:root:load_day (2011-06-20, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-20 PHX_Heat_Test/PHX_summer_2011_99p #11\n", + "INFO:root:load_day (2010-08-06, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-30, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-13, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-07-05, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-05 NYC_Heat_Test/NYC_summer_2000_01p #12\n", + "INFO:root:load_day (2017-05-25, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-08-07, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2011-07-16, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2010-06-22, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-08-03, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2009-08-13, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-13 PHX_Heat_Test/PHX_summer_2009_50p #13\n", + "INFO:root:load_day (2017-07-21, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2009-08-11, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-11 PHX_Heat_Test/PHX_summer_2009_50p #14\n", + "INFO:root:load_day (2009-07-24, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-24 PHX_Heat_Test/PHX_summer_2009_50p #15\n", + "INFO:root:load_day (2011-06-28, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-08-24, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-24 NYC_Heat_Test/NYC_summer_2015_50p #16\n", + "INFO:root:load_day (2008-05-30, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-06-07, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2008-08-04, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-06-01, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2000-06-27, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-07-28, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-24, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-08-13, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-15, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2020-08-29, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2009-05-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2008-06-12, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-07-27, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-27, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-04, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-03, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-19, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-06-25, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-06-25 NYC_Heat_Test/NYC_summer_2000_01p #17\n", + "INFO:root:load_day (2018-06-16, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2020-07-12, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-06-14, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-06-19, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-08-03, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-07-30, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2017-07-01, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-08-01, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-19, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-19 PHX_Heat_Test/PHX_summer_2009_50p #18\n", + "INFO:root:load_day (2017-08-16, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-05, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-19, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2015-07-06, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2009-08-20, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-20 PHX_Heat_Test/PHX_summer_2009_50p #19\n", + "INFO:root:load_day (2017-07-20, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-06-04, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-06-07, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-19, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-10, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-17, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-08-07, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-06-08, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2020-07-25, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2018-08-15, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-02, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-06-21, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2010-06-04, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-26, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-06-11, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-06-17, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-17 PHX_Heat_Test/PHX_summer_2011_99p #20\n", + "INFO:root:load_day (2017-05-29, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-30, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2000-07-06, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-06 NYC_Heat_Test/NYC_summer_2000_01p #21\n", + "INFO:root:load_day (2010-05-26, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-21, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2017-07-04, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-08-03, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-13, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-06-03, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-04, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-06-01, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-29, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-05-30, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2000-06-04, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-06-28, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-07-14, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-08-08, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-05-31, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-05-27, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2009-06-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-26 PHX_Heat_Test/PHX_summer_2009_50p #22\n", + "INFO:root:load_day (2008-06-16, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-07-25, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-07-20, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-06-29, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-29 PHX_Heat_Test/PHX_summer_2009_50p #23\n", + "INFO:root:load_day (2008-07-18, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-07-20, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-20 PHX_Heat_Test/PHX_summer_2009_50p #24\n", + "INFO:root:load_day (2009-07-28, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-28 PHX_Heat_Test/PHX_summer_2009_50p #25\n", + "INFO:root:load_day (2015-05-28, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-06-03, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-25, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-25 NYC_Heat_Test/NYC_summer_2015_50p #26\n", + "INFO:root:load_day (2020-08-07, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-05-29, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-07-16, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-07-15, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-15 NYC_Heat_Test/NYC_summer_2000_01p #27\n", + "INFO:root:load_day (2015-07-30, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-06-04, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-06-18, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-07-14, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-10, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-05-24, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2017-06-21, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-06-15, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2010-06-11, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-05, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-07-10, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-07-11, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-06-09, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-07-20, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2011-07-13, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-06-09, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-08-11, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-03, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-07-11, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-22, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-05-30, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2018-05-25, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-06-18, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-07-09, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-06-27, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-29, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-06-25, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-06-08, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-27, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-05, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-06-30, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-06-30 NYC_Heat_Test/NYC_summer_2000_01p #28\n", + "INFO:root:load_day (2010-06-29, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-07-03, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-03 PHX_Heat_Test/PHX_summer_2009_50p #29\n", + "INFO:root:load_day (2020-06-10, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2000-08-31, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-31 NYC_Heat_Test/NYC_summer_2000_01p #30\n", + "INFO:root:load_day (2018-06-14, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2017-07-11, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-08-03, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-07-25, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-07-25 NYC_Heat_Test/NYC_summer_2015_50p #31\n", + "INFO:root:load_day (2018-08-10, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2020-06-04, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2009-08-18, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-18 PHX_Heat_Test/PHX_summer_2009_50p #32\n", + "INFO:root:load_day (2008-05-24, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "2024-12-14 00:17:30.414030: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n", + "Traceback (most recent call last):\n", + "\n", + " File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n", + " ret = func(*args)\n", + " ^^^^^^^^^^^\n", + "\n", + " File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + "\n", + " File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n", + " raise TypeError(\n", + "\n", + "TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n", + "\n", + "\n", + "INFO:root:load_day (2020-07-30, PHX_Heat_Test/PHX_summer_2020_01p)\n" + ] + }, + { + "ename": "InvalidArgumentError", + "evalue": "{{function_node __wrapped__IteratorGetNext_output_types_4_device_/job:localhost/replica:0/task:0/device:CPU:0}} TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\nTraceback (most recent call last):\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n ret = func(*args)\n ^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n raise TypeError(\n\nTypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n\n\n\t [[{{node PyFunc}}]] [Op:IteratorGetNext] name: ", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m num_samples \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mval_ds\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_samples\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mspatiotemporal\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNumber of samples:\u001b[39m\u001b[38;5;124m\"\u001b[39m, num_samples)\n", + "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/iterator_ops.py:810\u001b[0m, in \u001b[0;36mOwnedIterator.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 808\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__next__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 809\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 810\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_internal\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 811\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m errors\u001b[38;5;241m.\u001b[39mOutOfRangeError:\n\u001b[1;32m 812\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m\n", + "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/iterator_ops.py:773\u001b[0m, in \u001b[0;36mOwnedIterator._next_internal\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 770\u001b[0m \u001b[38;5;66;03m# TODO(b/77291417): This runs in sync mode as iterators use an error status\u001b[39;00m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;66;03m# to communicate that there is no more data to iterate over.\u001b[39;00m\n\u001b[1;32m 772\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m context\u001b[38;5;241m.\u001b[39mexecution_mode(context\u001b[38;5;241m.\u001b[39mSYNC):\n\u001b[0;32m--> 773\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[43mgen_dataset_ops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miterator_get_next\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 774\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_iterator_resource\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 775\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_types\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_flat_output_types\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 776\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_shapes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_flat_output_shapes\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 778\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 779\u001b[0m \u001b[38;5;66;03m# Fast path for the case `self._structure` is not a nested structure.\u001b[39;00m\n\u001b[1;32m 780\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_element_spec\u001b[38;5;241m.\u001b[39m_from_compatible_tensor_list(ret) \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n", + "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/gen_dataset_ops.py:3029\u001b[0m, in \u001b[0;36miterator_get_next\u001b[0;34m(iterator, output_types, output_shapes, name)\u001b[0m\n\u001b[1;32m 3027\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _result\n\u001b[1;32m 3028\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m _core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m-> 3029\u001b[0m \u001b[43m_ops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from_not_ok_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3030\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m _core\u001b[38;5;241m.\u001b[39m_FallbackException:\n\u001b[1;32m 3031\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n", + "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/tensorflow/python/framework/ops.py:5883\u001b[0m, in \u001b[0;36mraise_from_not_ok_status\u001b[0;34m(e, name)\u001b[0m\n\u001b[1;32m 5881\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mraise_from_not_ok_status\u001b[39m(e, name) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m NoReturn:\n\u001b[1;32m 5882\u001b[0m e\u001b[38;5;241m.\u001b[39mmessage \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m name: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mstr\u001b[39m(name \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[0;32m-> 5883\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_status_to_exception(e) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "\u001b[0;31mInvalidArgumentError\u001b[0m: {{function_node __wrapped__IteratorGetNext_output_types_4_device_/job:localhost/replica:0/task:0/device:CPU:0}} TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\nTraceback (most recent call last):\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n ret = func(*args)\n ^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n raise TypeError(\n\nTypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n\n\n\t [[{{node PyFunc}}]] [Op:IteratorGetNext] name: " + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2009-08-23, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-23 PHX_Heat_Test/PHX_summer_2009_50p #33\n", + "INFO:root:load_day (2008-07-03, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-08-06, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-29, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-16, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-16 PHX_Heat_Test/PHX_summer_2009_50p #34\n", + "INFO:root:load_day (2015-07-05, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-06-09, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-11, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-08-25, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-25 NYC_Heat_Test/NYC_summer_2000_01p #35\n", + "INFO:root:load_day (2011-08-09, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2020-08-15, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2000-06-10, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2008-08-25, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-08-17, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-13, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2000-06-22, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-08-06, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-26, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2010-07-01, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-03, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-08-05, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-07-03, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-07-01, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-01 PHX_Heat_Test/PHX_summer_2011_99p #36\n", + "INFO:root:load_day (2020-05-25, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-06-20, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-06-13, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2008-08-17, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-08-27, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-06, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-06-17, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2009-07-09, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-09 PHX_Heat_Test/PHX_summer_2009_50p #37\n", + "INFO:root:load_day (2010-08-14, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-07-02, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-07-28, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-06-21, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-05-30, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-18, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2020-07-04, PHX_Heat_Test/PHX_summer_2020_01p)\n" + ] + } + ], + "source": [ + "num_samples = 0\n", + "for batch in val_ds:\n", + " num_samples += batch[0]['spatiotemporal'].shape[0]\n", + "print(\"Number of samples:\", num_samples)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2017-05-24, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-21, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-07-29, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-29 PHX_Heat_Test/PHX_summer_2009_50p #38\n", + "INFO:root:load_day (2020-08-13, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-13 PHX_Heat_Test/PHX_summer_2020_01p #39\n", + "INFO:root:load_day (2008-05-29, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-07-16, PHX_Heat_Test/PHX_summer_2008_25p)\n" + ] + } + ], + "source": [ + "# Initialize the Atmo Model\n", + "model_params = AtmoModelParams()\n", + "model = AtmoModel(model_params)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Split shape: (None, 50, 50, 5, 2, 256)\n", + "Separated shape (pre-slice): (None, 50, 50, 10, 256)\n", + "Reduced shape after removing outermost time steps: (None, 50, 50, 8, 256)\n", + "Split shape: (None, 50, 50, 5, 2, 256)\n", + "Separated shape (pre-slice): (None, 50, 50, 10, 256)\n", + "Reduced shape after removing outermost time steps: (None, 50, 50, 8, 256)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2000-08-15, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "2024-12-14 00:19:22.290293: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:961] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inatmo_conv_lstm_1/conv_lstm/conv_lstm2d_1/while/body/_1/atmo_conv_lstm_1/conv_lstm/conv_lstm2d_1/while/dropout_7/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2008-07-25, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-05-29, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2018-07-20, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2011-05-28, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2011-06-07, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-07-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-26 PHX_Heat_Test/PHX_summer_2009_50p #40\n", + "INFO:root:load_day (2011-08-03, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-07-15, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-10, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-23, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-07-17, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-08-28, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-08-02, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-06-25, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-25 PHX_Heat_Test/PHX_summer_2009_50p #1\n", + "INFO:root:load_day (2011-07-12, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-07-24, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2011-07-12 PHX_Heat_Test/PHX_summer_2011_99p #41\n", + "INFO:root:load_day (2010-07-04, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-31, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-06-22, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-30, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-08-11, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2008-08-23, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-10, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-07-06, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-07-18, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2011-08-24, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-08-24 PHX_Heat_Test/PHX_summer_2011_99p #2\n", + "INFO:root:load_day (2008-06-14, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "WARNING:root:incomplete data!: 2009-07-18 PHX_Heat_Test/PHX_summer_2009_50p #42\n", + "INFO:root:load_day (2000-08-23, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-23 NYC_Heat_Test/NYC_summer_2000_01p #43\n", + "INFO:root:load_day (2018-06-15, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-06-07, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-07-25, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-25 PHX_Heat_Test/PHX_summer_2011_99p #3\n", + "INFO:root:load_day (2015-07-12, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-06-30, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-15, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-06-07, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2020-06-07, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-25, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-07-17, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-06-23, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-15, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-08-08, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-20, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-06-17, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2020-06-11, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-12, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-08-24, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-04, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-07-03, NYC_Heat_Test/NYC_summer_2018_75p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 1/Unknown - 110s 110s/step - loss: 18692.8574 - mean_absolute_error: 69.4132 - root_mean_squared_error: 136.7218Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2011-06-26, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-26 PHX_Heat_Test/PHX_summer_2011_99p #4\n", + "INFO:root:load_day (2020-07-21, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-06-23, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-07-05, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2008-07-28, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-02, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-02 PHX_Heat_Test/PHX_summer_2009_50p #5\n", + "INFO:root:load_day (2015-07-31, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-07-15, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-15 PHX_Heat_Test/PHX_summer_2011_99p #6\n", + "INFO:root:load_day (2000-08-18, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-18 NYC_Heat_Test/NYC_summer_2000_01p #7\n", + "INFO:root:load_day (2010-07-27, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-05-31, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-04, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-07-03, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-08-17, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-06-12, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-07-18, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-08-28, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-28 NYC_Heat_Test/NYC_summer_2015_50p #8\n", + "INFO:root:load_day (2010-05-31, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-08-23, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2017-08-23 NYC_Heat_Test/NYC_summer_2017_25p #9\n", + "INFO:root:load_day (2011-06-27, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-08-16, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-16 NYC_Heat_Test/NYC_summer_2000_01p #10\n", + "INFO:root:load_day (2008-08-06, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-07-18, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-08-12, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-12 PHX_Heat_Test/PHX_summer_2009_50p #11\n", + "INFO:root:load_day (2015-08-01, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-08-07, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-23, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-09, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-07-08, NYC_Heat_Test/NYC_summer_2018_75p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 2/Unknown - 165s 55s/step - loss: 18722.3418 - mean_absolute_error: 70.0883 - root_mean_squared_error: 136.8296 Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2015-05-27, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-30, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-08-19, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-07-26, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-06-05, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-05-29, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-06-08, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-07-30, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-08-22, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-06-15, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-07-14, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-19, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-07-31, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-07-08, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2020-08-01, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-01 PHX_Heat_Test/PHX_summer_2020_01p #12\n", + "INFO:root:load_day (2015-07-25, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-08-18, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-08-22, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2017-05-26, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-08-25, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2017-08-25 NYC_Heat_Test/NYC_summer_2017_25p #13\n", + "INFO:root:load_day (2017-08-26, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-18, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-17, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2020-06-15, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2018-06-24, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2009-06-12, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2000-07-28, NYC_Heat_Test/NYC_summer_2000_01p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 3/Unknown - 219s 55s/step - loss: 18619.2109 - mean_absolute_error: 70.1765 - root_mean_squared_error: 136.4522Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:incomplete data!: 2000-07-28 NYC_Heat_Test/NYC_summer_2000_01p #14\n", + "INFO:root:load_day (2018-07-14, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-06-12, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2000-07-23, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2000-08-24, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-24 NYC_Heat_Test/NYC_summer_2000_01p #15\n", + "INFO:root:load_day (2020-08-22, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-22 PHX_Heat_Test/PHX_summer_2020_01p #16\n", + "INFO:root:load_day (2017-07-10, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-26, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-08-04, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-21, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2010-06-28, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-07-16, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-07-10, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-08-22, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-22 NYC_Heat_Test/NYC_summer_2000_01p #17\n", + "INFO:root:load_day (2015-05-31, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2009-08-15, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-15 PHX_Heat_Test/PHX_summer_2009_50p #18\n", + "INFO:root:load_day (2009-07-16, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-16 PHX_Heat_Test/PHX_summer_2009_50p #19\n", + "INFO:root:load_day (2015-05-29, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-08-12, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-08-28, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-28 NYC_Heat_Test/NYC_summer_2000_01p #20\n", + "INFO:root:load_day (2011-06-25, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-25 PHX_Heat_Test/PHX_summer_2011_99p #21\n", + "INFO:root:load_day (2020-08-31, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2011-07-10, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-08-14, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-14 NYC_Heat_Test/NYC_summer_2000_01p #22\n", + "INFO:root:load_day (2009-08-10, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-10 PHX_Heat_Test/PHX_summer_2009_50p #23\n", + "INFO:root:load_day (2015-08-16, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-06-19, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-07, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-07-16, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-08-30, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-05-24, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2008-06-30, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-01, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-06-11, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-06-17, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2000-07-24, NYC_Heat_Test/NYC_summer_2000_01p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 4/Unknown - 286s 59s/step - loss: 18606.8711 - mean_absolute_error: 70.4873 - root_mean_squared_error: 136.4070Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2015-07-24, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-31, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-08, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-06-28, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-28 PHX_Heat_Test/PHX_summer_2009_50p #24\n", + "INFO:root:load_day (2020-05-24, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-07-15, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-06, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-08-22, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "WARNING:root:incomplete data!: 2017-08-22 NYC_Heat_Test/NYC_summer_2017_25p #25\n", + "INFO:root:load_day (2017-06-19, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-05-25, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2000-07-22, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-22 NYC_Heat_Test/NYC_summer_2000_01p #26\n", + "INFO:root:load_day (2000-08-02, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-02 NYC_Heat_Test/NYC_summer_2000_01p #27\n", + "INFO:root:load_day (2011-08-11, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-08-09, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-09 NYC_Heat_Test/NYC_summer_2000_01p #28\n", + "INFO:root:load_day (2000-08-19, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-19 NYC_Heat_Test/NYC_summer_2000_01p #29\n", + "INFO:root:load_day (2000-07-17, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-17 NYC_Heat_Test/NYC_summer_2000_01p #30\n", + "INFO:root:load_day (2020-08-30, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-08-09, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-07-21, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2018-08-25, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2010-06-06, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-07-16, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-07-09, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-05-25, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-06-20, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-06-13, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-05-28, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-07-22, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-08-13, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-26 PHX_Heat_Test/PHX_summer_2009_50p #31\n", + "INFO:root:load_day (2000-08-01, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-01 NYC_Heat_Test/NYC_summer_2000_01p #32\n", + "INFO:root:load_day (2010-06-18, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-07-10, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-07-08, PHX_Heat_Test/PHX_summer_2015_75p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 5/Unknown - 351s 60s/step - loss: 18571.3164 - mean_absolute_error: 70.3885 - root_mean_squared_error: 136.2766Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2009-08-14, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-14 PHX_Heat_Test/PHX_summer_2009_50p #33\n", + "INFO:root:load_day (2015-06-25, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-16, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-06-19, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-08-14, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2010-07-24, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-08-01, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-26, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-06-29, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-08-15, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-07, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-21, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2018-08-14, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2010-07-21, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-11, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2010-08-05, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-07-23, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-08-13, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2011-06-10, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2017-06-15, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-07-07, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2009-07-30, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-30 PHX_Heat_Test/PHX_summer_2009_50p #34\n", + "INFO:root:load_day (2015-07-31, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2018-05-30, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2020-06-20, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2011-07-28, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-08-18, NYC_Heat_Test/NYC_summer_2015_50p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 6/Unknown - 405s 59s/step - loss: 18536.2461 - mean_absolute_error: 70.4138 - root_mean_squared_error: 136.1479Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:incomplete data!: 2015-08-18 NYC_Heat_Test/NYC_summer_2015_50p #35\n", + "INFO:root:load_day (2020-07-05, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-07-10, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-06-18, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-06-04, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-08-31, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-08-22, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-08-23, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-23 PHX_Heat_Test/PHX_summer_2020_01p #36\n", + "INFO:root:load_day (2018-07-28, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-05-28, NYC_Heat_Test/NYC_summer_2018_75p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 7/Unknown - 425s 52s/step - loss: 18526.6934 - mean_absolute_error: 70.4302 - root_mean_squared_error: 136.1128Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-12-14 00:26:23.180302: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 6239347564768653613\n", + "2024-12-14 00:26:23.180349: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 11923509177280531107\n", + "2024-12-14 00:26:23.180363: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 16364299007938407365\n", + "2024-12-14 00:26:23.180373: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 6472366602594991502\n", + "2024-12-14 00:26:23.180387: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 3410171957599719354\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Split shape: (None, 50, 50, 5, 2, 256)\n", + "Separated shape (pre-slice): (None, 50, 50, 10, 256)\n", + "Reduced shape after removing outermost time steps: (None, 50, 50, 8, 256)\n", + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2015-06-18, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2020-08-09, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-09, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-02, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-05-24, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-08-30, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-30 PHX_Heat_Test/PHX_summer_2009_50p #1\n", + "INFO:root:load_day (2011-06-24, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-05-31, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-08-08, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-08 PHX_Heat_Test/PHX_summer_2009_50p #2\n", + "INFO:root:load_day (2008-08-19, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-07-27, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-27 PHX_Heat_Test/PHX_summer_2009_50p #3\n", + "INFO:root:load_day (2015-07-01, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-14, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2018-07-23, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2010-08-28, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-07-06, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-06 PHX_Heat_Test/PHX_summer_2009_50p #4\n", + "INFO:root:load_day (2000-06-11, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-08-09, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-08-08, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-08 NYC_Heat_Test/NYC_summer_2000_01p #5\n", + "INFO:root:load_day (2000-07-25, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2018-06-22, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-13, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-07-31, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-31 PHX_Heat_Test/PHX_summer_2009_50p #6\n", + "INFO:root:load_day (2010-07-23, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-07-31, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2009-08-09, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-09 PHX_Heat_Test/PHX_summer_2009_50p #7\n", + "INFO:root:load_day (2020-08-27, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-27 PHX_Heat_Test/PHX_summer_2020_01p #8\n", + "INFO:root:load_day (2009-08-03, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-03 PHX_Heat_Test/PHX_summer_2009_50p #9\n", + "INFO:root:load_day (2000-06-05, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-06-16, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-12, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2020-05-26, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-06-02, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2009-05-24, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2011-07-07, PHX_Heat_Test/PHX_summer_2011_99p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2009-08-21, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-21 PHX_Heat_Test/PHX_summer_2009_50p #10\n", + "INFO:root:load_day (2011-06-20, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-20 PHX_Heat_Test/PHX_summer_2011_99p #11\n", + "INFO:root:load_day (2010-08-06, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-30, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-13, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-07-05, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-05 NYC_Heat_Test/NYC_summer_2000_01p #12\n", + "INFO:root:load_day (2017-05-25, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-08-07, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2011-07-16, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2010-06-22, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2018-08-03, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2009-08-13, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-13 PHX_Heat_Test/PHX_summer_2009_50p #13\n", + "INFO:root:load_day (2017-07-21, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2009-08-11, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-11 PHX_Heat_Test/PHX_summer_2009_50p #14\n", + "INFO:root:load_day (2009-07-24, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-24 PHX_Heat_Test/PHX_summer_2009_50p #15\n", + "INFO:root:load_day (2011-06-28, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-08-24, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-24 NYC_Heat_Test/NYC_summer_2015_50p #16\n", + "INFO:root:load_day (2008-05-30, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-06-07, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2008-08-04, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-06-01, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2000-06-27, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-07-28, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-24, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-08-13, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-15, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2020-08-29, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2009-05-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2008-06-12, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-07-27, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2017-06-27, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-04, NYC_Heat_Test/NYC_summer_2015_50p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2015-08-03, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-19, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-06-25, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-06-25 NYC_Heat_Test/NYC_summer_2000_01p #17\n", + "INFO:root:load_day (2018-06-16, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2020-07-12, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-06-14, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-06-19, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-08-03, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-07-30, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2017-07-01, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-08-01, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-19, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-19 PHX_Heat_Test/PHX_summer_2009_50p #18\n", + "INFO:root:load_day (2017-08-16, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-06-05, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-19, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2015-07-06, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2009-08-20, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-20 PHX_Heat_Test/PHX_summer_2009_50p #19\n", + "INFO:root:load_day (2017-07-20, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-06-04, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-06-07, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-19, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-10, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-17, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2008-08-07, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-06-08, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2020-07-25, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2018-08-15, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-02, NYC_Heat_Test/NYC_summer_2015_50p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2000-06-21, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2010-06-04, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-26, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-06-11, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-06-17, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-06-17 PHX_Heat_Test/PHX_summer_2011_99p #20\n", + "INFO:root:load_day (2017-05-29, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-30, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2000-07-06, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-06 NYC_Heat_Test/NYC_summer_2000_01p #21\n", + "INFO:root:load_day (2010-05-26, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-21, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2017-07-04, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-08-03, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-13, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-06-03, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-04, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-06-01, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-06-29, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-05-30, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2000-06-04, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-06-28, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-07-14, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-08-08, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-05-31, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-05-27, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2009-06-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-26 PHX_Heat_Test/PHX_summer_2009_50p #22\n", + "INFO:root:load_day (2008-06-16, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2017-07-25, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2008-07-20, PHX_Heat_Test/PHX_summer_2008_25p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2009-06-29, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-06-29 PHX_Heat_Test/PHX_summer_2009_50p #23\n", + "INFO:root:load_day (2008-07-18, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-07-20, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-20 PHX_Heat_Test/PHX_summer_2009_50p #24\n", + "INFO:root:load_day (2009-07-28, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-28 PHX_Heat_Test/PHX_summer_2009_50p #25\n", + "INFO:root:load_day (2015-05-28, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-06-03, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-25, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-08-25 NYC_Heat_Test/NYC_summer_2015_50p #26\n", + "INFO:root:load_day (2020-08-07, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-05-29, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-07-16, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-07-15, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-07-15 NYC_Heat_Test/NYC_summer_2000_01p #27\n", + "INFO:root:load_day (2015-07-30, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-06-04, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-06-18, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-07-14, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-10, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-05-24, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2017-06-21, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2000-06-15, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2010-06-11, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-08-05, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-07-10, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-07-11, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-06-09, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-07-20, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2011-07-13, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-06-09, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2017-08-11, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-06-03, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-07-11, NYC_Heat_Test/NYC_summer_2015_50p)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shape: TensorShape([None, 8, 50, 50, 256])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2008-07-22, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-05-30, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2018-05-25, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-06-18, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-07-09, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2018-06-27, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2015-08-29, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-06-25, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2010-06-08, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-27, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2008-07-05, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-06-30, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-06-30 NYC_Heat_Test/NYC_summer_2000_01p #28\n", + "INFO:root:load_day (2010-06-29, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-07-03, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-03 PHX_Heat_Test/PHX_summer_2009_50p #29\n", + "INFO:root:load_day (2020-06-10, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2000-08-31, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-31 NYC_Heat_Test/NYC_summer_2000_01p #30\n", + "INFO:root:load_day (2018-06-14, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2017-07-11, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2020-08-03, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-07-25, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "WARNING:root:incomplete data!: 2015-07-25 NYC_Heat_Test/NYC_summer_2015_50p #31\n", + "INFO:root:load_day (2018-08-10, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2020-06-04, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2009-08-18, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-18 PHX_Heat_Test/PHX_summer_2009_50p #32\n", + "INFO:root:load_day (2008-05-24, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "2024-12-14 00:33:05.440887: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n", + "Traceback (most recent call last):\n", + "\n", + " File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n", + " ret = func(*args)\n", + " ^^^^^^^^^^^\n", + "\n", + " File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + "\n", + " File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n", + " raise TypeError(\n", + "\n", + "TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n", + "\n", + "\n", + "2024-12-14 00:33:05.441216: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 1057787908657746217\n", + "2024-12-14 00:33:05.441253: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 14795421904330655973\n", + "2024-12-14 00:33:05.441266: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 3932962834327346845\n", + "2024-12-14 00:33:05.441279: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 7404725130195908535\n", + "2024-12-14 00:33:05.441317: I tensorflow/core/framework/local_rendezvous.cc:421] Local rendezvous recv item cancelled. Key hash: 8548369628431922430\n", + "INFO:root:load_day (2020-07-30, PHX_Heat_Test/PHX_summer_2020_01p)\n" + ] + }, + { + "ename": "InvalidArgumentError", + "evalue": "Graph execution error:\n\nDetected at node PyFunc defined at (most recent call last):\n\nDetected at node PyFunc defined at (most recent call last):\n\n2 root error(s) found.\n (0) INVALID_ARGUMENT: TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\nTraceback (most recent call last):\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n ret = func(*args)\n ^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n raise TypeError(\n\nTypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n\n\n\t [[{{node PyFunc}}]]\n\t [[IteratorGetNext]]\n\t [[IteratorGetNext/_2]]\n (1) INVALID_ARGUMENT: TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\nTraceback (most recent call last):\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n ret = func(*args)\n ^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n raise TypeError(\n\nTypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n\n\n\t [[{{node PyFunc}}]]\n\t [[IteratorGetNext]]\n0 successful operations.\n0 derived errors ignored. [Op:__inference_test_function_36678]", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[13], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Train the model\u001b[39;00m\n\u001b[1;32m 2\u001b[0m tb_callback \u001b[38;5;241m=\u001b[39m keras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mTensorBoard(log_dir\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./logs\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_ds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_ds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mtb_callback\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/climateiq-cnn/usl_models/usl_models/atmo_ml/model.py:126\u001b[0m, in \u001b[0;36mAtmoModel.fit\u001b[0;34m(self, train_dataset, val_dataset, epochs, steps_per_epoch, early_stopping, callbacks)\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m early_stopping \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 120\u001b[0m callbacks\u001b[38;5;241m.\u001b[39mappend(\n\u001b[1;32m 121\u001b[0m keras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mEarlyStopping(\n\u001b[1;32m 122\u001b[0m monitor\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mloss\u001b[39m\u001b[38;5;124m\"\u001b[39m, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmin\u001b[39m\u001b[38;5;124m\"\u001b[39m, patience\u001b[38;5;241m=\u001b[39mearly_stopping\n\u001b[1;32m 123\u001b[0m )\n\u001b[1;32m 124\u001b[0m )\n\u001b[0;32m--> 126\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 127\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrain_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 128\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 129\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 130\u001b[0m \u001b[43m \u001b[49m\u001b[43msteps_per_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msteps_per_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 131\u001b[0m \u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 132\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback..error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 67\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m 68\u001b[0m \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n", + "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/tensorflow/python/eager/execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 52\u001b[0m ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 53\u001b[0m tensors \u001b[38;5;241m=\u001b[39m pywrap_tfe\u001b[38;5;241m.\u001b[39mTFE_Py_Execute(ctx\u001b[38;5;241m.\u001b[39m_handle, device_name, op_name,\n\u001b[1;32m 54\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 56\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mInvalidArgumentError\u001b[0m: Graph execution error:\n\nDetected at node PyFunc defined at (most recent call last):\n\nDetected at node PyFunc defined at (most recent call last):\n\n2 root error(s) found.\n (0) INVALID_ARGUMENT: TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\nTraceback (most recent call last):\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n ret = func(*args)\n ^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n raise TypeError(\n\nTypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n\n\n\t [[{{node PyFunc}}]]\n\t [[IteratorGetNext]]\n\t [[IteratorGetNext/_2]]\n (1) INVALID_ARGUMENT: TypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\nTraceback (most recent call last):\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/ops/script_ops.py\", line 270, in __call__\n ret = func(*args)\n ^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/autograph/impl/api.py\", line 643, in wrapper\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n\n File \"/opt/conda/lib/python3.11/site-packages/tensorflow/python/data/ops/from_generator_op.py\", line 235, in generator_py_func\n raise TypeError(\n\nTypeError: `generator` yielded an element of shape (4, 200, 200, 12) where an element of shape (6, 200, 200, 12) was expected.\n\n\n\t [[{{node PyFunc}}]]\n\t [[IteratorGetNext]]\n0 successful operations.\n0 derived errors ignored. [Op:__inference_test_function_36678]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:load_day (2009-08-23, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-23 PHX_Heat_Test/PHX_summer_2009_50p #33\n", + "INFO:root:load_day (2008-07-03, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2015-08-06, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2008-07-29, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2009-08-16, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-08-16 PHX_Heat_Test/PHX_summer_2009_50p #34\n", + "INFO:root:load_day (2015-07-05, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2020-06-09, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2015-08-11, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2000-08-25, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-25 NYC_Heat_Test/NYC_summer_2000_01p #35\n", + "INFO:root:load_day (2011-08-09, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2020-08-15, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2000-06-10, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2008-08-25, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-08-17, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-13, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2000-06-22, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2015-08-06, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2015-08-26, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2010-07-01, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2015-07-03, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2017-08-05, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-07-03, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-07-01, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-01 PHX_Heat_Test/PHX_summer_2011_99p #36\n", + "INFO:root:load_day (2020-05-25, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-06-20, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2018-06-13, NYC_Heat_Test/NYC_summer_2018_75p)\n", + "INFO:root:load_day (2008-08-17, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-08-27, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2011-08-06, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2000-06-17, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2009-07-09, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-09 PHX_Heat_Test/PHX_summer_2009_50p #37\n", + "INFO:root:load_day (2010-08-14, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2010-07-02, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2020-07-28, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2008-06-21, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2010-05-30, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2009-06-18, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2020-07-04, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "INFO:root:load_day (2017-05-24, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2015-08-21, NYC_Heat_Test/NYC_summer_2015_50p)\n", + "INFO:root:load_day (2009-07-29, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-29 PHX_Heat_Test/PHX_summer_2009_50p #38\n", + "INFO:root:load_day (2020-08-13, PHX_Heat_Test/PHX_summer_2020_01p)\n", + "WARNING:root:incomplete data!: 2020-08-13 PHX_Heat_Test/PHX_summer_2020_01p #39\n", + "INFO:root:load_day (2008-05-29, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2008-07-16, PHX_Heat_Test/PHX_summer_2008_25p)\n", + "INFO:root:load_day (2000-08-15, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "INFO:root:load_day (2009-05-29, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "INFO:root:load_day (2011-05-28, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-07-26, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-26 PHX_Heat_Test/PHX_summer_2009_50p #40\n", + "INFO:root:load_day (2011-08-03, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2015-08-10, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2015-07-17, PHX_Heat_Test/PHX_summer_2015_75p)\n", + "INFO:root:load_day (2011-08-02, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2011-07-12, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "WARNING:root:incomplete data!: 2011-07-12 PHX_Heat_Test/PHX_summer_2011_99p #41\n", + "INFO:root:load_day (2010-07-04, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2017-06-22, NYC_Heat_Test/NYC_summer_2017_25p)\n", + "INFO:root:load_day (2010-08-11, NYC_Heat_Test/NYC_summer_2010_99p)\n", + "INFO:root:load_day (2011-08-10, PHX_Heat_Test/PHX_summer_2011_99p)\n", + "INFO:root:load_day (2009-07-18, PHX_Heat_Test/PHX_summer_2009_50p)\n", + "WARNING:root:incomplete data!: 2009-07-18 PHX_Heat_Test/PHX_summer_2009_50p #42\n", + "INFO:root:load_day (2000-08-23, NYC_Heat_Test/NYC_summer_2000_01p)\n", + "WARNING:root:incomplete data!: 2000-08-23 NYC_Heat_Test/NYC_summer_2000_01p #43\n", + "INFO:root:load_day (2018-06-15, NYC_Heat_Test/NYC_summer_2018_75p)\n" + ] + } + ], + "source": [ + "# Train the model\n", + "tb_callback = keras.callbacks.TensorBoard(log_dir=\"./logs\")\n", + "model.fit(train_ds, val_ds, epochs=1, callbacks=[tb_callback])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"atmo_conv_lstm\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " embedding (Embedding) multiple 488 \n", + " \n", + " spatial_cnn (Sequential) (None, 50, 50, 128) 252992 \n", + " \n", + " spatiotemporal_cnn (Sequen (None, None, 50, 50, 64 30480 \n", + " tial) ) \n", + " \n", + " conv_lstm (Sequential) (None, None, 50, 50, 51 45877248 \n", + " 2) \n", + " \n", + " t2_output_cnn (Sequential) (None, None, 200, 200, 69729 \n", + " 1) \n", + " \n", + " rh2_output_cnn (Sequential (None, None, 200, 200, 69729 \n", + " ) 1) \n", + " \n", + " wspd10_output_cnn (Sequent (None, None, 200, 200, 69729 \n", + " ial) 1) \n", + " \n", + " wdir10_output_cnn (Sequent (None, None, 200, 200, 69746 \n", + " ial) 2) \n", + " \n", + "=================================================================\n", + "Total params: 46440141 (177.16 MB)\n", + "Trainable params: 46440141 (177.16 MB)\n", + "Non-trainable params: 0 (0.00 Byte)\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model._model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'train_dataset' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Test calling the model on some training data\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m inputs, labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[43mtrain_dataset\u001b[49m))\n\u001b[1;32m 3\u001b[0m prediction \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mcall(inputs)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPrediction shape:\u001b[39m\u001b[38;5;124m\"\u001b[39m, prediction\u001b[38;5;241m.\u001b[39mshape)\n", + "\u001b[0;31mNameError\u001b[0m: name 'train_dataset' is not defined" + ] + } + ], + "source": [ + "# Test calling the model on some training data\n", + "inputs, labels = next(iter(train_dataset))\n", + "prediction = model.call(inputs)\n", + "print(\"Prediction shape:\", prediction.shape)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/usl_models/scripts/run_atmo_training_job.py b/usl_models/scripts/run_atmo_training_job.py new file mode 100644 index 00000000..d025e977 --- /dev/null +++ b/usl_models/scripts/run_atmo_training_job.py @@ -0,0 +1,87 @@ +import argparse +import distutils.core +import os +import pathlib + +from google.cloud import aiplatform +from google.cloud import storage # type:ignore[attr-defined] + +IMAGE = "us-docker.pkg.dev/vertex-ai/training/tf-gpu.2-14.py310:latest" + + +def main(): + cli_args = _parse_args() + + # Build a source distribution for the usl_models package. + this_file = pathlib.Path(os.path.realpath(__file__)) + package_dir = this_file.parent.parent + setup_path = package_dir / "setup.py" + distutils.core.run_setup(setup_path, script_args=["sdist", "--format=gztar"]) + + # Upload the usl_models package to GCS. + source_dist_path = package_dir / "dist" / "usl_models-0.0.0.tar.gz" + client = storage.Client(project="climateiq") + bucket = client.bucket("climateiq-vertexai") + bucket.blob("usl_models-0.0.0.tar.gz").upload_from_filename(str(source_dist_path)) + + # Run the training script trainer/flood_task.py in VertexAI. + job = aiplatform.CustomPythonPackageTrainingJob( + display_name=cli_args.model_name, + python_package_gcs_uri="gs://climateiq-vertexai/usl_models-0.0.0.tar.gz", + python_module_name="trainer.atmo_task", + container_uri=IMAGE, + model_serving_container_image_uri=IMAGE, + staging_bucket="gs://climateiq-vertexai", + ) + + job_args = ["--sim-names", *cli_args.sim_names] + if cli_args.epochs: + job_args.extend(("--epochs", str(cli_args.epochs))) + if cli_args.batch_size: + job_args.extend(("--batch-size", str(cli_args.batch_size))) + if cli_args.model_name: + job_args.extend(("--model-name", cli_args.model_name)) + + print(f"Creating training job with arguments {job_args}") + job.run( + model_display_name="atmo-model", + args=job_args, + replica_count=1, + machine_type="a2-highgpu-1g", + accelerator_type="NVIDIA_TESLA_A100", + accelerator_count=1, + sync=True, + ) + + +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-name", + dest="model_name", + type=str, + help="A name for the model.", + required=True, + ) + parser.add_argument("--epochs", dest="epochs", type=int, help="Number of epochs.") + parser.add_argument( + "--batch-size", dest="batch_size", type=int, help="Size of a batch." + ) + parser.add_argument( + "--sim-names", + dest="sim_names", + nargs="+", + type=str, + required=True, + help=( + "Space-separated set of simulations to train the model against, e.g. " + "--sim-names " + "NYC_Heat_Test/NYC_summer_ " + "Phoenix_Heat_Test/Phoenix_summer_" + ), + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/usl_models/tests/atmo_ml/atmo_model_test.py b/usl_models/tests/atmo_ml/atmo_model_test.py index e39f566e..345d2290 100644 --- a/usl_models/tests/atmo_ml/atmo_model_test.py +++ b/usl_models/tests/atmo_ml/atmo_model_test.py @@ -7,10 +7,10 @@ from usl_models.atmo_ml import constants from usl_models.atmo_ml import model_params -_TEST_MAP_HEIGHT = 100 -_TEST_MAP_WIDTH = 100 -_TEST_SPATIAL_FEATURES = 17 # lu_index is now separate -_TEST_SPATIOTEMPORAL_FEATURES = 9 +_TEST_MAP_HEIGHT = 200 +_TEST_MAP_WIDTH = 200 +_TEST_SPATIAL_FEATURES = 22 # lu_index is now separate +_TEST_SPATIOTEMPORAL_FEATURES = 12 _LU_INDEX_VOCAB_SIZE = 61 _EMBEDDING_DIM = 8 diff --git a/usl_models/tests/atmo_ml/atmo_utils_test.py b/usl_models/tests/atmo_ml/atmo_utils_test.py index ae7a3154..cc7e1dbc 100644 --- a/usl_models/tests/atmo_ml/atmo_utils_test.py +++ b/usl_models/tests/atmo_ml/atmo_utils_test.py @@ -62,10 +62,6 @@ def test_split_time_step_pairs(): expected_output = tf.constant( [ [ - [ - [[0, 1], [4, 5]], - [[8, 9], [12, 13]], - ], [ [[2, 3], [6, 7]], [[10, 11], [14, 15]], @@ -82,10 +78,6 @@ def test_split_time_step_pairs(): [[32, 33], [36, 37]], [[40, 41], [44, 45]], ], - [ - [[34, 35], [38, 39]], - [[42, 43], [46, 47]], - ], ], ] ) diff --git a/usl_models/tests/atmo_ml/dataset_test.py b/usl_models/tests/atmo_ml/dataset_test.py new file mode 100644 index 00000000..655a3e0b --- /dev/null +++ b/usl_models/tests/atmo_ml/dataset_test.py @@ -0,0 +1,106 @@ +# TODO: fix this file +import io + +from unittest import mock +from unittest.mock import MagicMock + +import numpy as np + +import usl_models.testing +from usl_models.atmo_ml import dataset +from usl_models.atmo_ml import constants + + +def create_mock_blob(data, dtype=np.float32, allow_pickle=True): + """Create a mock blob with simulated data and return it.""" + blob = MagicMock() + buf = io.BytesIO() + np.save(buf, data.astype(dtype), allow_pickle=allow_pickle) + buf.seek(0) + blob.open.return_value = buf + return blob + + +class TestAtmoMLDataset(usl_models.testing.TestCase): + @mock.patch("google.cloud.storage.Client") + def test_load_dataset_structure(self, mock_storage_client): + """Test creating AtmoML dataset from GCS with expected structure and shapes.""" + # Mock GCS client and bucket + mock_storage_client_instance = mock_storage_client.return_value + mock_bucket = MagicMock() + mock_storage_client_instance.bucket.return_value = mock_bucket + + num_days = 4 + timesteps_per_day = 6 + num_timesteps = num_days * timesteps_per_day + batch_size = 2 + + B = batch_size + H, W = constants.MAP_HEIGHT, constants.MAP_WIDTH + F_S = constants.NUM_SAPTIAL_FEATURES + F_ST = constants.NUM_SPATIOTEMPORAL_FEATURES + C = constants.OUTPUT_CHANNELS + T_I, T_O = constants.INPUT_TIME_STEPS, constants.OUTPUT_TIME_STEPS + + # Simulate mock blobs for datasets + mock_spatial_blob = create_mock_blob( + np.random.rand(H, W, F_S).astype(np.float32) + ) + mock_spatiotemporal_tensor = np.random.rand(H, W, F_ST).astype(np.float32) + mock_spatiotemporal_blobs = [ + create_mock_blob(mock_spatiotemporal_tensor) for _ in range(num_timesteps) + ] + mock_lu_index_blob = create_mock_blob( + np.random.randint( + low=0, + high=10, + size=(H, W), + ).astype(np.int32) + ) + mock_label_blobs = [ + create_mock_blob(np.random.rand(H, W, C).astype(np.float32)) + for _ in range(num_timesteps) + ] + + # Mock blob listing behavior to simulate folder structure + mock_bucket.list_blobs.side_effect = lambda prefix: { + "sim1/spatial": [mock_spatial_blob], + "sim1/spatiotemporal": mock_spatiotemporal_blobs, + "sim1/lu_index": [mock_lu_index_blob], + "sim1": mock_label_blobs, + }[prefix] + + # Define bucket names and folder paths + data_bucket_name = "test-data-bucket" + label_bucket_name = "test-label-bucket" + + # Call the function under test + ds = dataset.load_dataset( + data_bucket_name=data_bucket_name, + label_bucket_name=label_bucket_name, + sim_names=["sim1"], + timesteps_per_day=timesteps_per_day, + storage_client=mock_storage_client_instance, + ) + ds = ds.batch(batch_size=batch_size) + + inputs, labels = zip(*ds) + num_batches = num_days // batch_size + self.assertShapesRecursive( + list(inputs), + [ + { + "spatiotemporal": (B, T_I, H, W, F_ST), + "spatial": (B, H, W, F_S), + "lu_index": (B, H, W), + } + ] + * num_batches, + ) + self.assertShapesRecursive( + list(labels), + [ + (B, T_O, H, W, C), + ] + * num_batches, + ) diff --git a/usl_models/tests/atmo_ml/datasets_test.py b/usl_models/tests/atmo_ml/datasets_test.py deleted file mode 100644 index 96428e26..00000000 --- a/usl_models/tests/atmo_ml/datasets_test.py +++ /dev/null @@ -1,194 +0,0 @@ -import io -from unittest import mock -import numpy as np -from google.cloud import storage, firestore # type: ignore -from usl_models.atmo_ml import datasets -from usl_models.atmo_ml import cnn_inputs_outputs - - -@mock.patch.object(datasets, "create_atmo_dataset") -@mock.patch("usl_models.atmo_ml.datasets.make_predictions") -def test_atmo_dataset(mock_make_predictions, mock_create_dataset) -> None: - """Create expected dataset for the atmo model from GCS and Firestore objects.""" - mock_storage_client = mock.MagicMock(spec=storage.Client) - mock_firestore_client = mock.MagicMock(spec=firestore.Client) - - # Set parameters for the atmo dataset - bucket_name = "our-bucket-name" - spatiotemporal_file_names = [ - "spatiotemporal_data_1.npy", - "spatiotemporal_data_2.npy", - ] - label_file_names = ["labels_1.npy", "labels_2.npy"] - spatial_file_name = "spatial_data.npy" - lu_index_file_name = "lu_index.npy" - time_steps_per_day = 6 - batch_size = 4 - - # Set up mock data for the GCS - stinp_1 = np.random.rand(batch_size, 100, 100, 1) - labelinput_1 = np.random.rand(batch_size, 100, 100, 1) - stinp_2 = np.random.rand(batch_size, 100, 100, 1) - labelinput_2 = np.random.rand(batch_size, 100, 100, 1) - - # Call the function to process inputs and labels - mock_spatiotemporal_data_1, mock_labels_1 = cnn_inputs_outputs.divide_into_days( - stinp_1, labelinput_1, input_steps_per_day=4, label_steps_per_day=8 - ) - mock_spatiotemporal_data_2, mock_labels_2 = cnn_inputs_outputs.divide_into_days( - stinp_2, labelinput_2, input_steps_per_day=4, label_steps_per_day=8 - ) - mock_spatial_data = np.random.rand(100, 100, 1) - mock_lu_index_data = np.random.randint(1, 62, (100, 100)) - - def mock_blob_func(blob_name): - """Returns the mock data depending on the blob requested.""" - if blob_name == "spatiotemporal_data_1.npy": - mock_data = mock_spatiotemporal_data_1 - elif blob_name == "spatiotemporal_data_2.npy": - mock_data = mock_spatiotemporal_data_2 - elif blob_name == "labels_1.npy": - mock_data = mock_labels_1 - elif blob_name == "labels_2.npy": - mock_data = mock_labels_2 - elif blob_name == "spatial_data.npy": - mock_data = mock_spatial_data - elif blob_name == "lu_index.npy": - mock_data = mock_lu_index_data - else: - raise ValueError(f"Unexpected name {blob_name} passed to mock.") - - buf = io.BytesIO() - np.save(buf, mock_data) - buf.seek(0) - - mock_blob = mock.MagicMock(spec=storage.Blob) - mock_blob.open.return_value = buf - return mock_blob - - # Set the storage client to return the right mock data for each GCS path. - mock_storage_client.bucket().blob.side_effect = mock_blob_func - - # Mock Firestore interactions - mock_firestore_client.collection().document().get().to_dict.return_value = { - "some_firestore_key": "some_value" - } - - # Create the dataset using mocked data - dataset = datasets.create_atmo_dataset( - bucket_name=bucket_name, - spatiotemporal_file_names=spatiotemporal_file_names, - label_file_names=label_file_names, - spatial_file_name=spatial_file_name, - lu_index_file_name=lu_index_file_name, - time_steps_per_day=time_steps_per_day, - batch_size=batch_size, - storage_client=mock_storage_client, - firestore_client=mock_firestore_client, # Add Firestore client here - ) - - # Mock predictions returned by the model - mock_predictions = np.random.rand(batch_size, time_steps_per_day, 100, 100, 1) - mock_make_predictions.return_value = mock_predictions - - # Call the prediction function and validate - predictions = datasets.make_predictions(mock.MagicMock(), dataset) - assert predictions.shape == mock_predictions.shape, "Predictions shape mismatch" - - # Iterate through the dataset and check the shape - for data in dataset: - inputs, labels = data - assert inputs["spatiotemporal"].shape == ( - batch_size, - time_steps_per_day, - 100, - 100, - 1, - ), "Input spatiotemporal shape mismatch" - assert inputs["spatial"].shape == (100, 100, 1), "Spatial data shape mismatch" - assert inputs["lu_index"].shape == (100 * 100,), "Land use index shape mismatch" - assert labels.shape == ( - batch_size, - time_steps_per_day, - 100, - 100, - 1, - ), "Label shape mismatch" - - print("Inputs:", inputs) - print("Labels:", labels) - print("Predictions:", predictions) - - -@mock.patch("usl_models.atmo_ml.datasets.storage.Client") -def test_load_prediction_dataset(mock_storage_client) -> None: - """Test loading and generating batches for predictions.""" - bucket_name = "test-bucket" - spatiotemporal_file_names = [ - "spatiotemporal_data_1.npy", - "spatiotemporal_data_2.npy", - ] - spatial_file_name = "spatial_data.npy" - lu_index_file_name = "lu_index.npy" - batch_size = 4 - time_steps_per_day = 6 - - # Mock data for GCS - spatiotemporal_data_1 = np.random.rand(8, 100, 100, 1) - spatiotemporal_data_2 = np.random.rand(8, 100, 100, 1) - spatial_data = np.random.rand(100, 100, 1) - lu_index_data = np.random.randint(1, 62, (100, 100)) - - def mock_blob_func(blob_name): - """Returns the mock data depending on the blob requested.""" - if blob_name == "spatiotemporal_data_1.npy": - mock_data = spatiotemporal_data_1 - elif blob_name == "spatiotemporal_data_2.npy": - mock_data = spatiotemporal_data_2 - elif blob_name == "spatial_data.npy": - mock_data = spatial_data - elif blob_name == "lu_index.npy": - mock_data = lu_index_data - else: - raise ValueError(f"Unexpected name {blob_name} passed to mock.") - - buf = io.BytesIO() - np.save(buf, mock_data) - buf.seek(0) - - mock_blob = mock.MagicMock(spec=storage.Blob) - mock_blob.open.return_value = buf - return mock_blob - - # Set the storage client to return the right mock data for each GCS path. - mock_storage_client().bucket().blob.side_effect = mock_blob_func - - # Load prediction dataset and iterate through the generator - dataset_generator = datasets.load_prediction_dataset( - bucket_name=bucket_name, - spatiotemporal_file_names=spatiotemporal_file_names, - spatial_file_name=spatial_file_name, - lu_index_file_name=lu_index_file_name, - batch_size=batch_size, - time_steps_per_day=time_steps_per_day, - storage_client=mock_storage_client(), - ) - - # Check that the generator yields correctly batched data - for i, inputs in enumerate(dataset_generator): - spatiotemporal_batch = inputs["spatiotemporal"] - spatial_batch = inputs["spatial"] - lu_index_batch = inputs["lu_index"] - - assert spatiotemporal_batch.shape == ( - batch_size, - 100, - 100, - 1, - ), "Spatiotemporal shape mismatch" - assert spatial_batch.shape == (100, 100, 1), "Spatial data shape mismatch" - assert lu_index_batch.shape == (100 * 100,), "LU index shape mismatch" - - print(f"Batch {i + 1} spatiotemporal shape: {spatiotemporal_batch.shape}") - print(f"Batch {i + 1} spatial shape: {spatial_batch.shape}") - print(f"Batch {i + 1} LU index shape: {lu_index_batch.shape}") diff --git a/usl_models/trainer/atmo_task.py b/usl_models/trainer/atmo_task.py new file mode 100644 index 00000000..2381a511 --- /dev/null +++ b/usl_models/trainer/atmo_task.py @@ -0,0 +1,181 @@ +r"""Run training on vertex AI for Atmo ML model. + +Command: +cd usl_models +python trainer/atmo_task.py +""" +import argparse +import logging +import os + +from google.cloud import firestore +from google.cloud import storage +import tensorflow as tf +from tensorflow.python.client import device_lib + +from usl_models.atmo_ml import model as atmo_model +from usl_models.atmo_ml import metastore +from usl_models.atmo_ml import model_params as atmo_model_params +from usl_models.atmo_ml import dataset + +logging.basicConfig(level=logging.INFO) + +# Argument parser +parser = argparse.ArgumentParser() +parser.add_argument( + "--model-name", dest="model_name", type=str, help="A name for the model." +) +parser.add_argument("--epochs", dest="epochs", type=int, help="Number of epochs.") +parser.add_argument( + "--batch-size", dest="batch_size", type=int, help="Size of a batch." +) +parser.add_argument( + "--model-dir", + dest="model_dir", + default=os.environ.get("AIP_MODEL_DIR"), + type=str, + help="Model dir.", +) +parser.add_argument( + "--distribute", + dest="distribute", + type=str, + default="single", + help="distributed training strategy", + choices=["single", "mirrored", "multiworker", "tpu"], +) +args = parser.parse_args() + +logging.info("DEVICES" + str(device_lib.list_local_devices())) + +# Define the distribution strategy +if args.distribute == "single": + if tf.test.is_gpu_available(): + strategy = tf.distribute.OneDeviceStrategy(device="/gpu:0") + else: + strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0") + logging.info("Single device training") +elif args.distribute == "mirrored": + strategy = tf.distribute.MirroredStrategy() + logging.info("Mirrored Strategy distributed training") +elif args.distribute == "multiworker": + strategy = tf.distribute.MultiWorkerMirroredStrategy() + logging.info("Multi-worker Strategy distributed training") + logging.info("TF_CONFIG = {}".format(os.environ.get("TF_CONFIG", "Not found"))) +elif args.distribute == "tpu": + cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu="local") + tf.config.experimental_connect_to_cluster(cluster_resolver) + tf.tpu.experimental.initialize_tpu_system(cluster_resolver) + strategy = tf.distribute.TPUStrategy(cluster_resolver) + print("All devices: ", tf.config.list_logical_devices("TPU")) + +logging.info("num_replicas_in_sync = {}".format(strategy.num_replicas_in_sync)) + + +def _is_chief(task_type, task_id): + """Checks for primary if multiworker training.""" + return ( + (task_type == "chief") + or (task_type == "worker" and task_id == 0) + or task_type is None + ) + + +def train( + model: atmo_model.AtmoModel, + train_dataset: tf.data.Dataset, + val_dataset: tf.data.Dataset, + sim_names: list[str] +) -> None: + """Trains a model with the given dataset and saves the model to GCS.""" + kwargs = {} + if args.epochs is not None: + kwargs["epochs"] = args.epochs + + model.fit(train_dataset, val_dataset=val_dataset, **kwargs) + + if args.distribute == "multiworker": + task_type, task_id = ( + strategy.cluster_resolver.task_type, + strategy.cluster_resolver.task_id, + ) + else: + task_type, task_id = None, None + + # Save model depending on distribution strategy + if args.distribute == "tpu": + save_locally = tf.saved_model.SaveOptions( + experimental_io_device="/job:localhost" + ) + logging.info("Saving model to %s", args.model_dir) + model_dir = args.model_dir + model.save_model(model_dir, options=save_locally) + elif _is_chief(task_type, task_id): + logging.info("Saving model to %s", args.model_dir) + model_dir = args.model_dir + model.save_model(model_dir) + else: + model_dir = args.model_dir + "/workertemp_" + str(task_id) + logging.info("Saving model to %s", model_dir) + tf.io.gfile.makedirs(model_dir) + model.save_model(model_dir) + + metastore.write_model_metadata( + firestore_client, + gcs_model_dir=model_dir, + sim_names=sim_names, + model_params=model._model_params, + epochs=args.epochs, + model_name=args.model_name, + ) + + +if __name__ == "__main__": + sim_dirs = [ + ('NYC_Heat_Test', [ + 'NYC_summer_2000_01p', + 'NYC_summer_2010_25p', + 'NYC_summer_2015_50p', + 'NYC_summer_2017_75p', + 'NYC_summer_2018_99p' + ]), + ('PHX_Heat_Test', [ + 'PHX_summer_2008_25p', + 'PHX_summer_2009_50p', + 'PHX_summer_2011_99p', + 'PHX_summer_2015_75p', + 'PHX_summer_2020_01p' + ]) + ] + + sim_names = [] + for sim_dir, subdirs in sim_dirs: + for subdir in subdirs: + sim_names.append(sim_dir + '/' + subdir) + + with strategy.scope(): + data_bucket_name = "climateiq-study-area-feature-chunks" + label_bucket_name = "climateiq-study-area-label-chunks" + time_steps_per_day = 6 + batch_size = 4 + firestore_client = firestore.Client(project="climateiq") + model_params = atmo_model_params.default_params() + if args.batch_size is not None: + model_params["batch_size"] = args.batch_size + model = atmo_model.AtmoModel(params=model_params) + logging.info( + "Training model for %s epochs with params %s", args.epochs, model_params + ) + storage_client = storage.Client(project="climateiq") + ds = dataset.load_dataset( + data_bucket_name=data_bucket_name, + label_bucket_name=label_bucket_name, + sim_names=sim_names, + timesteps_per_day=time_steps_per_day, + storage_client=storage_client + ).batch(batch_size=4) + train_dataset, val_dataset, test_dataset = dataset.split_dataset( + ds, train_frac=0.7, val_frac=0.15, test_frac=0.15 + ) + + train(model, train_dataset, val_dataset, sim_names) diff --git a/usl_models/trainer/flood_task.py b/usl_models/trainer/flood_task.py index 26db805a..228032ef 100644 --- a/usl_models/trainer/flood_task.py +++ b/usl_models/trainer/flood_task.py @@ -133,36 +133,37 @@ def train( ) -with strategy.scope(): - # Creation of dataset, and model building/compiling need to be within - # `strategy.scope()`. - firestore_client = firestore.Client(project="climateiq") - - model_params = usl_models.flood_ml.model_params.default_params() - if args.batch_size is not None: - model_params["batch_size"] = args.batch_size - model = usl_models.flood_ml.model.FloodModel(params=model_params) - logging.info( - "Training model for %s epochs with params %s", args.epochs, model_params - ) +if __name__ == "__main__": + with strategy.scope(): + # Creation of dataset, and model building/compiling need to be within + # `strategy.scope()`. + firestore_client = firestore.Client(project="climateiq") + + model_params = usl_models.flood_ml.model_params.default_params() + if args.batch_size is not None: + model_params["batch_size"] = args.batch_size + model = usl_models.flood_ml.model.FloodModel(params=model_params) + logging.info( + "Training model for %s epochs with params %s", args.epochs, model_params + ) - kwargs = {} - if args.batch_size is not None: - kwargs["batch_size"] = args.batch_size - train_dataset = usl_models.flood_ml.dataset.load_dataset_windowed( - sim_names=args.sim_names, - dataset_split="train", - firestore_client=firestore_client, - storage_client=storage.Client(project="climateiq"), - **kwargs, - ) - val_dataset = usl_models.flood_ml.dataset.load_dataset_windowed( - sim_names=args.sim_names, - dataset_split="val", - firestore_client=firestore_client, - storage_client=storage.Client(project="climateiq"), - **kwargs, - ) + kwargs = {} + if args.batch_size is not None: + kwargs["batch_size"] = args.batch_size + train_dataset = usl_models.flood_ml.dataset.load_dataset_windowed( + sim_names=args.sim_names, + dataset_split="train", + firestore_client=firestore_client, + storage_client=storage.Client(project="climateiq"), + **kwargs, + ) + val_dataset = usl_models.flood_ml.dataset.load_dataset_windowed( + sim_names=args.sim_names, + dataset_split="val", + firestore_client=firestore_client, + storage_client=storage.Client(project="climateiq"), + **kwargs, + ) -train(model, train_dataset, val_dataset, firestore_client) + train(model, train_dataset, val_dataset, firestore_client) diff --git a/usl_models/usl_models/atmo_ml/cnn_inputs_outputs.py b/usl_models/usl_models/atmo_ml/cnn_inputs_outputs.py index 3aef4b20..30a138dc 100644 --- a/usl_models/usl_models/atmo_ml/cnn_inputs_outputs.py +++ b/usl_models/usl_models/atmo_ml/cnn_inputs_outputs.py @@ -1,8 +1,18 @@ import tensorflow as tf -# Main function to divide inputs and labels by days -def divide_into_days(inputs, labels, input_steps_per_day=4, label_steps_per_day=8): +def divide_into_days(inputs, labels=None, input_steps_per_day=4, label_steps_per_day=8): + """Splits inputs (and labels if provided) into daily sequences. + + Args: + inputs (tf.Tensor): Input time series data. + labels (tf.Tensor, optional): Label time series data. + input_steps_per_day (int): Number of input steps per day. Defaults to 4. + label_steps_per_day (int): Number of label steps per day. Defaults to 8. + + Returns: + tuple: daily input tensors and, if labels are given, daily label tensors. + """ total_days = inputs.shape[0] // input_steps_per_day day_inputs_list = [] day_labels_list = [] @@ -16,25 +26,37 @@ def divide_into_days(inputs, labels, input_steps_per_day=4, label_steps_per_day= label_steps_per_day, ) day_inputs_list.append(day_inputs) - day_labels_list.append(day_labels) - # Print shapes for debugging - tf.print(f"Day {day_idx + 1} generated inputs shape:", tf.shape(day_inputs)) - tf.print(f"Day {day_idx + 1} generated labels shape:", tf.shape(day_labels)) - return day_inputs_list, day_labels_list + if labels is not None: + day_labels_list.append(day_labels) + # tf.print(f"Day {day_idx + 1} inputs shape:", tf.shape(day_inputs)) + if labels is not None: + pass + # tf.print(f"Day {day_idx + 1} labels shape:", tf.shape(day_labels)) + return day_inputs_list, day_labels_list if labels is not None else None -# Function to process inputs and labels for each day def process_day( inputs, labels, day_idx, total_days, input_steps_per_day, label_steps_per_day ): - # Get inputs for current day + """Retrieves input and label sequences for a specific day. + + Args: + inputs (tf.Tensor): Full input time series data. + labels (tf.Tensor, optional): Full label time series data. + day_idx (int): Index of the day. + total_days (int): Total number of days in data. + input_steps_per_day (int): Input steps per day. + label_steps_per_day (int): Label steps per day. + + Returns: + tuple: Day-specific input tensor and, if labels are provided, label tensor. + """ + # Get inputs for the day, with padding at start if needed if day_idx == 0: - # First day - no previous time step, duplicate the first value day_inputs = tf.concat( [inputs[0:1], inputs[0 : input_steps_per_day + 1]], axis=0 ) else: - # Intermediate and last days day_inputs = tf.concat( [ inputs[ @@ -48,12 +70,17 @@ def process_day( ], axis=0, ) + # Add padding at the end for the last day if needed if day_idx == total_days - 1: - # Last day - no next time step, duplicate the last value day_inputs = tf.concat([day_inputs, inputs[-1:]], axis=0) - # Get labels for current day - day_labels = labels[ - day_idx * label_steps_per_day : day_idx * label_steps_per_day - + label_steps_per_day - ] + + # Get label sequence if provided + day_labels = ( + labels[ + day_idx * label_steps_per_day : day_idx * label_steps_per_day + + label_steps_per_day + ] + if labels is not None + else None + ) return day_inputs, day_labels diff --git a/usl_models/usl_models/atmo_ml/constants.py b/usl_models/usl_models/atmo_ml/constants.py index 1c04c056..4f193ff8 100644 --- a/usl_models/usl_models/atmo_ml/constants.py +++ b/usl_models/usl_models/atmo_ml/constants.py @@ -7,11 +7,9 @@ # Other well-defined constants INPUT_TIME_STEPS = 6 OUTPUT_CHANNELS = 5 -OUTPUT_TIME_STEPS = 10 -num_spatial_features = 22 -num_spatiotemporal_features = 12 +OUTPUT_TIME_STEPS = 8 +NUM_SAPTIAL_FEATURES = 22 +NUM_SPATIOTEMPORAL_FEATURES = 12 TIME_STEPS_PER_DAY = 4 -NUM_DAYS = 500 -TOTAL_TIME_STEPS = TIME_STEPS_PER_DAY * NUM_DAYS -lu_index_vocab_size = 61 -embedding_dim = 8 +LU_INDEX_VOCAB_SIZE = 61 +EMBEDDING_DIM = 8 diff --git a/usl_models/usl_models/atmo_ml/data_utils.py b/usl_models/usl_models/atmo_ml/data_utils.py index 93973b93..d8b7e192 100644 --- a/usl_models/usl_models/atmo_ml/data_utils.py +++ b/usl_models/usl_models/atmo_ml/data_utils.py @@ -38,35 +38,35 @@ def boundary_pairs(data: tf.Tensor) -> tf.Tensor: def split_time_step_pairs(data: tf.Tensor) -> tf.Tensor: """Splits a tensor into two time steps. - Takes a time sequence of T paired tensors and returns 2T single tensors. - This function follows the work done by boundary_pairs, which groups tensors - tensors up, and splits them up again prior to retrieving the final output. - - Input: [[T0, T1], [T2, T3], [T4, T5]] - Output: [T0, T1, T2, T3, T4, T5] - - For more details about this function and boundary_pairs, see - https://www.notion.so/climate-iq/AtmoML-Architecture-Proposals-and-Design-c00d0e54265c4bb8a72ce01fd475f116?pvs=4#455d785c174a482c90a78efde380adf3. - Args: data: A time sequence of paired tensors of shape [B, T, H, W, 2C]. Returns: - A time sequence of single tensors of shape [B, 2T, H, W, C]. + A time sequence of single tensors of shape [B, 2T - 2, H, W, C]. """ - # Transpose to [B, H, W, T, 2C] so that we can split the channel dimension + # Transpose to [B, H, W, T, 2C] for easier manipulation permuted = tf.transpose(data, (0, 2, 3, 1, 4)) - # Get the dynamic shape batch_size = tf.shape(permuted)[0] height = tf.shape(permuted)[1] width = tf.shape(permuted)[2] - time_steps = tf.shape(permuted)[3] * 2 # Original time steps are doubled - channels = tf.shape(permuted)[4] // 2 # Original channels are halved - - # Reshape to [B, H, W, 2T, C] - reshaped = tf.reshape(permuted, (batch_size, height, width, time_steps, channels)) - - # Transpose back to [B, 2T, H, W, C] - output = tf.transpose(reshaped, (0, 3, 1, 2, 4)) + original_time_steps = tf.shape(permuted)[3] # This will be T + channels = tf.shape(permuted)[4] // 2 # Halve the channels (2C -> C) + # Split the time sequence into two halves for each pair + # Example: [ (X_{d-1}^{18}, X^0), (X^0, X^6),..] -> separate into individual tensors + split = tf.reshape( + permuted, (batch_size, height, width, original_time_steps, 2, channels) + ) + print("Split shape:", split.shape) + # Reshape into individual time steps + separated = tf.reshape( + split, (batch_size, height, width, original_time_steps * 2, channels) + ) + print("Separated shape (pre-slice):", separated.shape) + # Remove the outermost single time steps (X_{d-1}^{18} and X_{d+1}^0) + reduced = separated[:, :, :, 1:-1, :] + print("Reduced shape after removing outermost time steps:", reduced.shape) + # Transpose back to [B, T_adjusted, H, W, C] + output = tf.transpose(reduced, (0, 3, 1, 2, 4)) + tf.print("Output shape:", output.shape) return output diff --git a/usl_models/usl_models/atmo_ml/dataset.py b/usl_models/usl_models/atmo_ml/dataset.py new file mode 100644 index 00000000..ddc79fe5 --- /dev/null +++ b/usl_models/usl_models/atmo_ml/dataset.py @@ -0,0 +1,276 @@ +import logging +import random + +import functools +import hashlib # For hashing days +import tensorflow as tf +import numpy as np +from google.cloud import storage # type: ignore +from usl_models.atmo_ml import cnn_inputs_outputs, constants +from usl_models.shared import downloader +import re # To extract dates from filenames + + +# Load data from Google Cloud Storage +@functools.lru_cache(maxsize=512) +def load_pattern_from_cloud( + bucket_name: str, + prefix: str, + storage_client: storage.Client, + max_blobs: int | None = None, +) -> tf.Tensor: + """Download all files in the folder. + + If max_blobs is specified, only returns that many blobs. + """ + bucket = storage_client.bucket(bucket_name) + blobs = bucket.list_blobs(prefix=prefix) + all_data = [] + blob_count = 0 + for blob in blobs: + blob_count += 1 + if max_blobs is not None and blob_count > max_blobs: + break + + if blob.name.endswith(".npy"): # Ensure only .npy files are processed + all_data.append(downloader.blob_to_tensor(blob)) + + return tf.stack(all_data) + + +def get_date(filename: str) -> str: + return filename.split(".")[2].split("_")[0] + + +def hash_day(sim_name: str, date: str) -> float: + """Hash a timestamp into a float between 0 and 1. + + Ensure that all timestamps for the same day hash to the same value. + + Args: + filepath (str): A string in the format 'met_em.d03.2000-05-24_00:00:00.npy' + or 'wrfout_d03_2000-05-24_00:00:00.npy'. + + Returns: + float: A hash value between 0 and 1 for the day. + """ + # Hash the date part to ensure all timestamps for the same day are consistent + return ( + int(hashlib.sha256((sim_name + date).encode()).hexdigest(), 16) + % (10**8) + / (10**8) + ) + + +def get_all_simulation_days( + sim_names: list[str], storage_client, bucket_name: str +) -> list[tuple[str, str]]: + """Retrieve all simulation days from simulation names. + + Returns: sim_name, date + """ + all_days = set() + bucket = storage_client.bucket(bucket_name) + + for sim_name in sim_names: + # List blobs under the simulation folder + blobs = bucket.list_blobs(prefix=f"{sim_name}/") + + num_blobs = 0 + for blob in blobs: + num_blobs += 1 + # Extract the date from the filename (e.g., "2000-05-24.npy") + + filename = blob.name.split("/")[-1] + all_days.add((sim_name, get_date(filename))) + + assert num_blobs > 0 + + return sorted(all_days) + + +def load_dataset( + data_bucket_name: str, + label_bucket_name: str, + sim_names: list[str], + storage_client: storage.Client = None, + shuffle: bool = True, + hash_range=(0.0, 0.8), +) -> tuple[tf.data.Dataset, tf.data.Dataset]: + storage_client = storage_client or storage.Client() + + # Early validation + assert storage_client.bucket( + label_bucket_name + ).exists(), f"Bucket does not exist: {label_bucket_name}" + assert storage_client.bucket( + data_bucket_name + ).exists(), f"Bucket does not exist: {data_bucket_name}" + + sim_name_dates = get_all_simulation_days( + sim_names=sim_names, storage_client=storage_client, bucket_name=data_bucket_name + ) + + if shuffle: + random.shuffle(sim_name_dates) + + logging.info("sim_name_dates %s" % str(sim_name_dates)) + # for i, (sim_name, day) in enumerate(sim_name_dates): + + # print("all_days:", sim_name_dates) + + def data_generator(): + missing_files: int = 0 + for sim_name, day in sim_name_dates: + # Skip days not in this dataset + if hash_range[0] <= hash_day(sim_name, day) < hash_range[1]: + continue + + # Load spatial, LU index, spatiotemporal, and label from their folders + lu_index_data = load_pattern_from_cloud( + data_bucket_name, + f"{sim_name}/lu_index", + storage_client, + max_blobs=1, + )[0] + spatial_data = load_pattern_from_cloud( + data_bucket_name, f"{sim_name}/spatial", storage_client, max_blobs=1 + )[0] + + load_result = load_day( + day, + sim_name, + bucket_name_inputs=data_bucket_name, + bucket_name_labels=label_bucket_name, + storage_client=storage_client, + ) + if load_result is None: + missing_files += 1 + logging.warning( + "incomplete data!: %s %s #%d" % (day, sim_name, missing_files) + ) + continue + + spatiotemporal_data, label_data = load_result + + yield { + "spatiotemporal": spatiotemporal_data, + "spatial": spatial_data.numpy(), + "lu_index": lu_index_data.numpy().reshape(200, 200), + }, label_data + + dataset = tf.data.Dataset.from_generator( + lambda: data_generator(), + output_signature=( + { + "spatiotemporal": tf.TensorSpec( + shape=( + constants.INPUT_TIME_STEPS, + constants.MAP_HEIGHT, + constants.MAP_WIDTH, + constants.NUM_SPATIOTEMPORAL_FEATURES, + ), + dtype=tf.float32, + ), + "spatial": tf.TensorSpec( + shape=( + constants.MAP_HEIGHT, + constants.MAP_WIDTH, + constants.NUM_SAPTIAL_FEATURES, + ), + dtype=tf.float32, + ), + "lu_index": tf.TensorSpec( + shape=( + constants.MAP_HEIGHT, + constants.MAP_WIDTH, + ), + dtype=tf.int32, + ), + }, + tf.TensorSpec( + shape=( + constants.OUTPUT_TIME_STEPS, + constants.MAP_HEIGHT, + constants.MAP_WIDTH, + constants.OUTPUT_CHANNELS, + ), + dtype=tf.float32, + ), + ), + ) + return dataset + + +# Pad or truncate data to a target length along the first axis. +def pad_or_truncate_data(data, target_length, pad_value=0): + """Pad or truncate data to the target length along the first axis.""" + current_length = data.shape[0] + if current_length > target_length: + return data[:target_length] + elif current_length < target_length: + pad_shape = [target_length - current_length] + list(data.shape[1:]) + padding = np.full(pad_shape, pad_value, dtype=data.dtype) + return np.concatenate([data, padding], axis=0) + return data + + +def extract_dates( + bucket_name: str, path: str, storage_client: storage.Client +) -> list[str]: + """Extract unique dates from filenames in the given bucket path.""" + bucket = storage_client.bucket(bucket_name) + blobs = bucket.list_blobs(prefix=path) + + dates = set() + for blob in blobs: + match = re.search(r"\d{4}-\d{2}-\d{2}", blob.name) # Match YYYY-MM-DD format + if match: + dates.add(match.group()) + return sorted(dates) + + +def load_day( + day: str, # 2000-05-24 + sim_name: str, + bucket_name_inputs: str, + bucket_name_labels: str, + storage_client: storage.Client, +) -> tuple[tf.Tensor, tf.Tensor] | None: + # TODO: mock this out to return random tensor to get the count. + + logging.info("load_day (%s, %s)" % (day, sim_name)) + """Load spatiotemporal data and labels for a given day from GCP.""" + # Extract all available dates dynamically + base_path = f"{sim_name}/spatiotemporal/" + all_dates = extract_dates(bucket_name_inputs, base_path, storage_client) + + # Ensure the requested day exists in the list of dates + if day not in all_dates: + raise ValueError(f"Day {day} is not found in the dataset!") + + # Determine previous, current, and next days + day_index = all_dates.index(day) + previous_day = all_dates[day_index - 1] if day_index > 0 else day + next_day = all_dates[day_index + 1] if day_index < len(all_dates) - 1 else day + + # Load inputs for previous, current, and next days + def load_inputs_for_date(date: str, max_blobs: int | None = None): + input_path = f"{sim_name}/spatiotemporal/met_em.d03.{date}_" + return load_pattern_from_cloud( + bucket_name_inputs, input_path, storage_client, max_blobs + ) + + inputs_previous = load_inputs_for_date(previous_day, max_blobs=1) + inputs_current = load_inputs_for_date(day) + inputs_next = load_inputs_for_date(next_day, max_blobs=1) + + # Concatenate inputs to form a single tensor + inputs = tf.concat([inputs_previous, inputs_current, inputs_next], axis=0) + + # Load labels for the day + label_path = f"{sim_name}/wrfout_d03_{day}_" + labels = load_pattern_from_cloud(bucket_name_labels, label_path, storage_client) + num_label_timestamps = labels.shape[0] + if num_label_timestamps == constants.OUTPUT_TIME_STEPS: + return inputs, labels diff --git a/usl_models/usl_models/atmo_ml/datasets.py b/usl_models/usl_models/atmo_ml/datasets.py deleted file mode 100644 index 8f8d2f28..00000000 --- a/usl_models/usl_models/atmo_ml/datasets.py +++ /dev/null @@ -1,248 +0,0 @@ -import tensorflow as tf -import numpy as np -from google.cloud import storage, firestore # type: ignore -from usl_models.atmo_ml import constants -from usl_models.atmo_ml import cnn_inputs_outputs - - -# Load data from Google Cloud Storage with Firestore logging -def load_data_from_cloud( - bucket_name: str, - file_name: str, - storage_client: storage.Client, - firestore_client: firestore.Client = None, -): - """Load data from Google Cloud Storage, with optional Firestore logging.""" - bucket = storage_client.bucket(bucket_name) - blob = bucket.blob(file_name) - downloaded_data = blob.download_as_bytes() - - # Log progress to Firestore (if Firestore client is provided) - if firestore_client: - doc_ref = firestore_client.collection("dataset_loading").document(file_name) - doc_ref.set({"file_name": file_name, "status": "loaded"}) - - return np.load(downloaded_data) - - -def load_spatiotemporal_data_from_cloud( - bucket_name: str, - file_name: str, - storage_client: storage.Client, - firestore_client: firestore.Client = None, -) -> tf.Tensor: - """Load spatiotemporal data from Google Cloud Storage with Firestore logging.""" - data = load_data_from_cloud( - bucket_name, file_name, storage_client, firestore_client - ) - return tf.convert_to_tensor(data, dtype=tf.float32) - - -def load_labels_from_cloud( - bucket_name: str, - file_name: str, - storage_client: storage.Client, - firestore_client: firestore.Client = None, -) -> tf.Tensor: - """Load labels from Google Cloud Storage with Firestore logging.""" - labels = load_data_from_cloud( - bucket_name, file_name, storage_client, firestore_client - ) - return tf.convert_to_tensor(labels, dtype=tf.float32) - - -def load_lu_index_from_cloud( - bucket_name: str, - file_name: str, - storage_client: storage.Client, - firestore_client: firestore.Client = None, -) -> tf.Tensor: - """Load land use index from Google Cloud Storage with Firestore logging.""" - lu_index_data = load_data_from_cloud( - bucket_name, file_name, storage_client, firestore_client - ) - return tf.convert_to_tensor(lu_index_data, dtype=tf.int32) - - -def load_spatial_data_from_cloud( - bucket_name: str, - file_name: str, - storage_client: storage.Client, - firestore_client: firestore.Client = None, -) -> tf.Tensor: - """Load spatial data from Google Cloud Storage with Firestore logging.""" - spatial_data = load_data_from_cloud( - bucket_name, file_name, storage_client, firestore_client - ) - return tf.convert_to_tensor(spatial_data, dtype=tf.float32) - - -def load_prediction_dataset( - bucket_name: str, - spatiotemporal_file_names: list, - spatial_file_name: str, - lu_index_file_name: str, - batch_size: int, - time_steps_per_day: int, - storage_client: storage.Client, -): - """Load prediction data from GCS and generate batches for predictions. - - Args: - bucket_name: Name of the GCS bucket. - spatiotemporal_file_names: List of GCS paths for spatiotemporal. - spatial_file_name: GCS path for spatial data. - lu_index_file_name: GCS path for LU index. - batch_size: Batch size for prediction. - time_steps_per_day: Number of time steps per day for spatiotemporal data. - storage_client: The GCS client. - - Yields: - A batch of prediction inputs. - """ - bucket = storage_client.bucket(bucket_name) - - # Load spatial and LU index data - spatial_blob = bucket.blob(spatial_file_name) - lu_index_blob = bucket.blob(lu_index_file_name) - - spatial_data = np.load(spatial_blob.open("rb")) - lu_index_data = np.load(lu_index_blob.open("rb")) - - # Load spatiotemporal data in batches for prediction - for st_file_name in spatiotemporal_file_names: - spatiotemporal_blob = bucket.blob(st_file_name) - spatiotemporal_data = np.load(spatiotemporal_blob.open("rb")) - - # Split the spatiotemporal data into batches - for i in range(0, spatiotemporal_data.shape[0], batch_size): - batch_spatiotemporal = spatiotemporal_data[i : i + batch_size] - - # Yield the batch of inputs - inputs = { - "spatiotemporal": batch_spatiotemporal, - "spatial": spatial_data, - "lu_index": lu_index_data.flatten(), - } - yield inputs - - -def create_atmo_dataset( - bucket_name: str, - spatiotemporal_file_names: list[str], - label_file_names: list[str], - spatial_file_name: str, - lu_index_file_name: str, - time_steps_per_day: int, - batch_size: int = 32, - shuffle: bool = True, - storage_client: storage.Client = None, - firestore_client: firestore.Client = None, -) -> tf.data.Dataset: - """Creates the dataset for the AtmoML model with optional Firestore logging. - - Args: - bucket_name: The GCS bucket name. - spatiotemporal_file_names: List of file names for spatiotemporal in GCS. - label_file_names: List of file names for corresponding labels in the GCS. - spatial_file_name: File name for spatial features (static data) in the GCS. - lu_index_file_name: File name for the land use index data in the GCS. - time_steps_per_day: Number of time steps to divide per day. - batch_size: Batch size for the dataset. - shuffle: Whether to shuffle the dataset. - storage_client: An instance of Google Cloud Storage client. - firestore_client: An optional Firestore client to log data loading status. - - Returns: - A tf.data.Dataset object yielding input dictionaries compatible with AtmoML. - """ - - def data_generator(): - lu_index_data = load_lu_index_from_cloud( - bucket_name, lu_index_file_name, storage_client, firestore_client - ) - spatial_data = load_spatial_data_from_cloud( - bucket_name, spatial_file_name, storage_client, firestore_client - ) - - for spatiotemporal_file_name, label_file_name in zip( - spatiotemporal_file_names, label_file_names - ): - spatiotemporal_data = load_spatiotemporal_data_from_cloud( - bucket_name, spatiotemporal_file_name, storage_client, firestore_client - ) - label_data = load_labels_from_cloud( - bucket_name, label_file_name, storage_client, firestore_client - ) - - # Divide the spatiotemporal data and labels into days - inputs, labels = cnn_inputs_outputs.divide_into_days( - spatiotemporal_data, label_data, time_steps_per_day - ) - - # Yield each day's input and label as a batch - for day_inputs, day_labels in zip(inputs, labels): - yield { - "spatiotemporal": day_inputs, # shape: (t,h,w,f) - "spatial": spatial_data, # shape: (height, width, spatial_features) - "lu_index": lu_index_data, # shape: (height * width,) - }, day_labels # shape: (time_steps_per_day, height, width, 1) - - dataset = tf.data.Dataset.from_generator( - data_generator, - output_signature=( - { - "spatiotemporal": tf.TensorSpec( - shape=( - time_steps_per_day, - constants.MAP_HEIGHT, - constants.MAP_WIDTH, - constants.num_spatiotemporal_features, - ), - dtype=tf.float32, - ), - "spatial": tf.TensorSpec( - shape=( - constants.MAP_HEIGHT, - constants.MAP_WIDTH, - constants.num_spatial_features, - ), - dtype=tf.float32, - ), - "lu_index": tf.TensorSpec( - shape=(constants.MAP_HEIGHT * constants.MAP_WIDTH,), - dtype=tf.int32, - ), - }, - tf.TensorSpec( - shape=( - time_steps_per_day, - constants.MAP_HEIGHT, - constants.MAP_WIDTH, - 1, - ), - dtype=tf.float32, - ), - ), - ) - - if shuffle: - dataset = dataset.shuffle(buffer_size=len(spatiotemporal_file_names)) - - dataset = dataset.batch(batch_size) - - return dataset - - -def make_predictions(model: tf.keras.Model, dataset: tf.data.Dataset) -> np.ndarray: - """Make predictions using the AtmoML model on the provided dataset. - - Args: - model: The trained AtmoML model. - dataset: The dataset to predict on. - - Returns: - A numpy array of predictions. - """ - predictions = model.predict(dataset) - return predictions diff --git a/usl_models/usl_models/atmo_ml/input_output_sequences.py b/usl_models/usl_models/atmo_ml/input_output_sequences.py index 58cd343e..53db40eb 100644 --- a/usl_models/usl_models/atmo_ml/input_output_sequences.py +++ b/usl_models/usl_models/atmo_ml/input_output_sequences.py @@ -47,7 +47,6 @@ def create_input_output_sequences( # Input sequences for t in range(time_steps_per_day): - print(f"Time step: {t}") # Debug: Print the current time step if day == 0: # First day if t == 0: input_seq = tf.stack( diff --git a/usl_models/usl_models/atmo_ml/metastore.py b/usl_models/usl_models/atmo_ml/metastore.py new file mode 100644 index 00000000..4bda10ae --- /dev/null +++ b/usl_models/usl_models/atmo_ml/metastore.py @@ -0,0 +1,271 @@ +import datetime +from typing import Any, Sequence, TypeVar +import urllib.parse + +from google.cloud import firestore # type:ignore[attr-defined] + +import usl_models.atmo_ml.model_params + + +def write_model_metadata( + db: firestore.Client, + gcs_model_dir: str, + sim_names: Sequence[str], + model_params: usl_models.atmo_ml.model_params.AtmoModelParams, + epochs: int, + model_name: str, +) -> str: + """Writes information on a trained model to the metastore. + + Args: + db: The firestore client to use when retrieving metadata. + gcs_model_dir: The location of the saved model in GCS. + sim_names: The simulations on which the model was trained. + model_params: The parameters used to train the model. + epochs: The number of epochs the model was trained for. + model_name: A name to associate with the model. + + Returns: + The document ID of the written model metadata document. + """ + # Use the GCS location of the model as a unique value for the ID. + # URL-escape the ID, as characters such as / are not allowed in document IDs. + model_id = urllib.parse.quote(gcs_model_dir, safe=()) + db.collection("models").document(model_id).set( + { + "trained_on": [_get_simulation_doc(db, sim_name) for sim_name in sim_names], + "trained_at_utc": datetime.datetime.utcnow(), + "gcs_model_dir": gcs_model_dir, + "epochs": epochs, + "model_params": model_params, + "model_name": model_name, + } + ) + return model_id + + +def get_spatiotemporal_feature_chunk_metadata( + db: firestore.Client, sim_name: str +) -> dict[str, Any]: + """Retrieves metadata for spatiotemporal features in GCS. + + Args: + db: The firestore client to use when retrieving metadata. + sim_name: The simulation for which to retrieve metadata. + + Returns: + A dictionary with metadata for the spatiotemporal features. + + Raises: + ValueError: If a simulation `sim_name` cannot be found. + """ + sim = _get_simulation_doc(db, sim_name).get().to_dict() + if sim is None: + raise ValueError(f"No such simulation {sim_name} found.") + + return sim["spatiotemporal_features"] + + +def get_spatial_feature_chunk_metadata( + db: firestore.Client, sim_name: str +) -> list[dict[str, Any]]: + """Retrieves metadata for spatial features in GCS. + + Args: + db: The firestore client to use when retrieving metadata. + sim_name: The simulation for which to retrieve metadata. + + Returns: + sequence of dictionaries with keys stating location of spatial feature tensors. + + Raises: + ValueError: If a simulation `sim_name` cannot be found. + """ + sim = _get_simulation_doc(db, sim_name).get().to_dict() + if sim is None: + raise ValueError(f"No such simulation {sim_name} found.") + + study_area_ref = sim["study_area"] + return [ + doc.to_dict() for doc in study_area_ref.collection("spatial_chunks").stream() + ] + + +def get_lu_index_feature_chunk_metadata( + db: firestore.Client, sim_name: str +) -> list[dict[str, Any]]: + """Retrieves metadata for land use index features in GCS. + + Args: + db: The firestore client to use when retrieving metadata. + sim_name: The simulation for which to retrieve metadata. + + Returns: + sequence of dictionaries with keys stating location of lu_index feature tensors. + + Raises: + ValueError: If a simulation `sim_name` cannot be found. + """ + sim = _get_simulation_doc(db, sim_name).get().to_dict() + if sim is None: + raise ValueError(f"No such simulation {sim_name} found.") + + return sim["lu_index"] + + +def get_spatial_feature_and_label_metadata( + db: firestore.Client, sim_name: str, dataset_split: str +) -> list[tuple[dict[str, Any], dict[str, Any]]]: + """Retrieves metadata for the location of (feature, label) pairs in GCS. + + Args: + db: The Firestore client to use when retrieving metadata. + sim_name: The simulation for which to retrieve metadata. + dataset_split: Which dataset to retrieve: train, val, or test. + + Returns: + A sequence of tuples, where each feature is a dictionary with keys + 'spatial_matrix_path', 'spatiotemporal_matrix_path', and 'lu_index_path' stating + the GCS locations of the feature tensors. The label is a dictionary with key + 'gcs_uri' for the location of the label tensor. + + Raises: + ValueError: If the simulation `sim_name` cannot be found. + AssertionError: If the labels and spatial features for the simulation do not + contain the same set of chunks. + """ + # Retrieve all feature metadata + feature_metadata = get_all_feature_chunk_metadata(db, sim_name) + + # Retrieve all label chunks for the simulation. + label_chunks_collection = ( + _get_simulation_doc(db, sim_name) + .collection("label_chunks") + .where(filter=firestore.FieldFilter("dataset", "==", dataset_split)) + ) + label_metadata = [doc.to_dict() for doc in label_chunks_collection.stream()] + + # Map features and labels by their chunk indices. + features_by_chunk_index = { + (feature["x_index"], feature["y_index"]): feature + for feature in feature_metadata + } + labels_by_chunk_index = { + (label["x_index"], label["y_index"]): label for label in label_metadata + } + + # Ensure we have the same chunk indices for features and labels. + missing_features = _missing_keys(labels_by_chunk_index, features_by_chunk_index) + if missing_features: + raise AssertionError( + "Features and label chunks do not line up. " + f'Indices missing from features: {", ".join(map(str, missing_features))}' + ) + + # Return feature & matching label metadata associated with the same indices. + return [ + (features_by_chunk_index[index], label) + for index, label in labels_by_chunk_index.items() + ] + + +def get_all_feature_chunk_metadata( + db: firestore.Client, sim_name: str +) -> list[dict[str, Any]]: + """Retrieves metadata for all feature types (spatial, spatiotemporal, and lu_index). + + Args: + db: The Firestore client to use when retrieving metadata. + sim_name: The simulation for which to retrieve metadata. + + Returns: + A list of dictionaries, each containing metadata for a feature chunk. + """ + spatial_metadata = get_spatial_feature_chunk_metadata(db, sim_name) + spatiotemporal_metadata = get_spatiotemporal_feature_chunk_metadata(db, sim_name) + lu_index_metadata = get_lu_index_feature_chunk_metadata(db, sim_name) + + # Combine metadata for all feature types into a single structure. + all_metadata = [] + for spatial, spatiotemporal, lu_index in zip( + spatial_metadata, spatiotemporal_metadata, lu_index_metadata + ): + all_metadata.append( + { + "x_index": spatial["x_index"], + "y_index": spatial["y_index"], + "spatial_matrix_path": spatial["feature_matrix_path"], + "spatiotemporal_matrix_path": spatiotemporal["feature_matrix_path"], + "lu_index_path": lu_index["feature_matrix_path"], + } + ) + + return all_metadata + + +def get_feature_chunk_metadata_for_prediction( + db: firestore.Client, study_area: str +) -> list[dict[str, Any]]: + """Retrieves metadata for all feature chunks in the given study area. + + Args: + db: The Firestore client to use when retrieving metadata. + study_area: The name of the study area, e.g., NYC. + + Returns: + A list of dictionaries, each representing a chunk's metadata, with paths for + spatial, spatiotemporal, and lu_index features. + + Raises: + ValueError: If the study area cannot be found or it has no chunks. + """ + # Retrieve the study area document + doc_ref = db.collection("study_areas").document(study_area) + doc = doc_ref.get() + + if not doc.exists: + raise ValueError(f"No such study area '{study_area}' found.") + + # Retrieve all chunks within the study area + chunks = doc_ref.collection("chunks").stream() + chunk_metadata = [chunk.to_dict() for chunk in chunks] + + if not chunk_metadata: + raise ValueError(f"No chunks found in study area '{study_area}'.") + + # Add paths for all feature types + combined_metadata = [] + for chunk in chunk_metadata: + combined_metadata.append( + { + "x_index": chunk["x_index"], + "y_index": chunk["y_index"], + "spatial_matrix_path": chunk.get("spatial_matrix_path"), + "spatiotemporal_matrix_path": chunk.get("spatiotemporal_matrix_path"), + "lu_index_path": chunk.get("lu_index_path"), + } + ) + + return combined_metadata + + +_T = TypeVar("_T") + + +def _missing_keys(d1: dict[_T, Any], d2: dict[_T, Any]) -> Sequence[_T]: + """Returns dictionary keys present in d1 but not d2.""" + return [key for key in d1.keys() if key not in d2] + + +def _get_simulation_doc( + db: firestore.Client, sim_name: str +) -> firestore.DocumentReference: + """Retrieves the firestore document for the simulation with the given name.""" + return db.collection("simulations").document( + # Quote the name to avoid characters not allowed in IDs such as slashes. + urllib.parse.quote( + # Unquote to support being passed both quoted & unquoted simulation names. + urllib.parse.unquote(sim_name), + safe=(), + ) + ) diff --git a/usl_models/usl_models/atmo_ml/model.py b/usl_models/usl_models/atmo_ml/model.py index c19e1180..9fdd5f45 100644 --- a/usl_models/usl_models/atmo_ml/model.py +++ b/usl_models/usl_models/atmo_ml/model.py @@ -33,10 +33,10 @@ def __init__( self, params: AtmoModelParams | None = None, spatial_dims: tuple[int, int] = (constants.MAP_HEIGHT, constants.MAP_WIDTH), - num_spatial_features: int = constants.num_spatial_features, - num_spatiotemporal_features: int = constants.num_spatiotemporal_features, - lu_index_vocab_size: int = constants.lu_index_vocab_size, - embedding_dim: int = constants.embedding_dim, + num_spatial_features: int = constants.NUM_SAPTIAL_FEATURES, + num_spatiotemporal_features: int = constants.NUM_SPATIOTEMPORAL_FEATURES, + lu_index_vocab_size: int = constants.LU_INDEX_VOCAB_SIZE, + embedding_dim: int = constants.EMBEDDING_DIM, ): """Creates the Atmo model. @@ -394,8 +394,19 @@ def call(self, inputs: AtmoModel.Input) -> tf.Tensor: Returns: A rank-5 tensor of all output predictions. """ - spatial_input = inputs["spatial"] + spatial_input = inputs["spatial"] # (B, H, W, C) st_input = inputs["spatiotemporal"] + lu_index_input = inputs["lu_index"] # lu_index is passed separately + + B = st_input.shape[0] + C = constants.NUM_SAPTIAL_FEATURES + H, W = constants.MAP_HEIGHT, constants.MAP_WIDTH + STF = constants.NUM_SPATIOTEMPORAL_FEATURES + T = constants.INPUT_TIME_STEPS + + tf.ensure_shape(spatial_input, (B, H, W, C)) + tf.ensure_shape(st_input, (B, T, H, W, STF)) + tf.ensure_shape(lu_index_input, (B, H, W)) # Ensure all spatial features are present; if not, replace with zeros if spatial_input.shape[-1] < self._spatial_features: @@ -414,7 +425,6 @@ def call(self, inputs: AtmoModel.Input) -> tf.Tensor: paddings=[[0, 0], [0, 0], [0, 0], [0, 0], [0, missing_channels]], constant_values=0, ) - lu_index_input = inputs["lu_index"] # lu_index is passed separately # Reshape lu_index matrix for embedding lu_index_input_flat = tf.reshape( @@ -422,7 +432,7 @@ def call(self, inputs: AtmoModel.Input) -> tf.Tensor: ) lu_index_embedded_flat = self.lu_index_embedding(lu_index_input_flat) - # Reshape back to matrix form + # Reshape back to matrix form (200, 200, ?, ?) lu_index_embedded = tf.reshape( lu_index_embedded_flat, (-1, self._spatial_height, self._spatial_width, self._embedding_dim), diff --git a/usl_models/usl_models/flood_ml/dataset.py b/usl_models/usl_models/flood_ml/dataset.py index c07a483c..2240392c 100644 --- a/usl_models/usl_models/flood_ml/dataset.py +++ b/usl_models/usl_models/flood_ml/dataset.py @@ -3,17 +3,15 @@ import logging import random from typing import Any, Iterator, Tuple -import urllib.parse from google.cloud import firestore # type:ignore[attr-defined] from google.cloud import storage # type:ignore[attr-defined] -import numpy -from numpy.typing import NDArray import tensorflow as tf from usl_models.flood_ml import constants from usl_models.flood_ml import metastore from usl_models.flood_ml import model +from usl_models.shared import downloader def load_dataset( @@ -327,7 +325,7 @@ def _generate_temporal_tensor( gcs_url = temporal_metadata["as_vector_gcs_uri"] logging.info("Retrieving temporal features from %s.", gcs_url) - temporal_vector = _download_as_tensor(storage_client, gcs_url) + temporal_vector = downloader.download_as_tensor(storage_client, gcs_url) temporal_vector = tf.transpose( tf.tile(tf.reshape(temporal_vector, (1, len(temporal_vector))), [m_rainfall, 1]) ) @@ -354,31 +352,13 @@ def _iter_geo_feature_label_tensors( logging.info( "Retrieving features from %s and labels from %s", feature_url, label_url ) - feature_tensor = _download_as_tensor(storage_client, feature_url) - label_tensor = _download_as_tensor(storage_client, label_url) + feature_tensor = downloader.download_as_tensor(storage_client, feature_url) + label_tensor = downloader.download_as_tensor(storage_client, label_url) reshaped_label_tensor = tf.transpose(label_tensor, perm=[2, 0, 1]) yield feature_tensor, reshaped_label_tensor -def _download_as_array(client: storage.Client, gcs_url: str) -> NDArray: - """Retrieves the contents at `gcs_url` from GCS as a numpy array.""" - parsed = urllib.parse.urlparse(gcs_url) - bucket = client.bucket(parsed.netloc) - blob = bucket.blob(parsed.path.lstrip("/")) - - with blob.open("rb") as fd: - return numpy.load(fd) - - -def _download_as_tensor(client: storage.Client, gcs_url: str) -> tf.Tensor: - """Retrieves the contents at `gcs_url` from GCS as a tf tensor.""" - return tf.convert_to_tensor( - _download_as_array(client, gcs_url), - dtype=tf.float32, - ) - - def _iter_model_inputs_for_prediction( firestore_client: firestore.Client, storage_client: storage.Client, @@ -435,7 +415,7 @@ def _iter_study_area_tensors( chunk_name = feature_url.split("/")[-1] logging.info("Retrieving features from %s ", feature_url) - feature_tensor = _download_as_tensor(storage_client, feature_url) + feature_tensor = downloader.download_as_tensor(storage_client, feature_url) yield feature_tensor, chunk_name diff --git a/usl_models/usl_models/shared/__init__.py b/usl_models/usl_models/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/usl_models/usl_models/shared/downloader.py b/usl_models/usl_models/shared/downloader.py new file mode 100644 index 00000000..65fb39b6 --- /dev/null +++ b/usl_models/usl_models/shared/downloader.py @@ -0,0 +1,35 @@ +import urllib.parse +import numpy as np +import numpy.typing as npt +from google.cloud import storage +import tensorflow as tf + + +def download_as_array(client: storage.Client, gcs_url: str) -> npt.NDArray: + """Retrieves the contents at `gcs_url` from GCS as a numpy array.""" + parsed = urllib.parse.urlparse(gcs_url) + bucket = client.bucket(parsed.netloc) + blob = bucket.blob(parsed.path.lstrip("/")) + + with blob.open("rb") as fd: + return np.load(fd) + + +def download_as_tensor(client: storage.Client, gcs_url: str) -> tf.Tensor: + """Retrieves the contents at `gcs_url` from GCS as a tf tensor.""" + return tf.convert_to_tensor( + download_as_array(client, gcs_url), + dtype=tf.float32, + ) + + +def blob_to_array(blob: storage.Blob) -> npt.NDArray: + with blob.open("rb") as fd: + return np.load(fd) + + +def blob_to_tensor(blob: storage.Blob): + return tf.convert_to_tensor( + blob_to_array(blob), + dtype=tf.float32, + ) diff --git a/usl_models/usl_models/testing/__init__.py b/usl_models/usl_models/testing/__init__.py new file mode 100644 index 00000000..684d1374 --- /dev/null +++ b/usl_models/usl_models/testing/__init__.py @@ -0,0 +1,30 @@ +import builtins +import unittest +from typing import Any +import numpy as np +import tensorflow as tf + + +class TestCase(unittest.TestCase): + """Testing utilss.""" + + def assertShapesRecursive(self, obj: Any, expected: Any, path: str = ""): + """Recursively checks the shapes of numpy arrays in a data structure.""" + if isinstance(obj, np.ndarray) or tf.is_tensor(obj): + self.assertEqual(obj.shape, expected, msg=f"Shape mismatch in path {path}") + return + + self.assertEqual(type(obj), type(expected), msg=f"Type mismatch in path {path}") + + match type(obj): + case builtins.list: + self.assertEqual(len(obj), len(expected)) + for i, (o, e) in enumerate(zip(obj, expected)): + self.assertShapesRecursive(o, e, path + f"[{i}]") + case builtins.tuple: + for i, (o, e) in enumerate(zip(obj, expected)): + self.assertShapesRecursive(o, e, path + f"[{i}]") + case builtins.dict: + assert set(obj.keys()) == set(expected.keys()) + for k in obj: + self.assertShapesRecursive(obj[k], expected[k], path + f"['{k}']")