Issue/79 gsp data

conftest.py

@@ -36,8 +36,8 @@ def sat_data_source(sat_filename: Path):
     return SatelliteDataSource(
         image_size_pixels=pytest.IMAGE_SIZE_PIXELS,
         filename=sat_filename,
-        history_len=0,
-        forecast_len=1,
+        history_minutes=0,
+        forecast_minutes=5,
         channels=('HRV', ),
         n_timesteps_per_batch=2,
         convert_to_numpy=True)

notebooks/2021-08-25/video.py → notebooks/2021-08/2021-08-25/video.py

@@ -8,7 +8,7 @@
 import cv2
 
 DATA_PATH = "gs://solar-pv-nowcasting-data/prepared_ML_training_data/v4/"
-TEMP_PATH = "."
+TEMP_PATH = ""
 
 # set up data generator
 train_dataset = NetCDFDataset(24_900, os.path.join(DATA_PATH, "train"), os.path.join(TEMP_PATH, "train"))

notebooks/2021-08-26/video.py → notebooks/2021-08/2021-08-26/video.py

@@ -19,7 +19,7 @@
 ##############
 
 DATA_PATH = "gs://solar-pv-nowcasting-data/prepared_ML_training_data/v4/"
-TEMP_PATH = "."
+TEMP_PATH = ""
 
 # set up data generator
 train_dataset = NetCDFDataset(24_900, os.path.join(DATA_PATH, "train"), os.path.join(TEMP_PATH, "train"))

notebooks/2021-09/2021-09-01/GSP data.ipynb

@@ -0,0 +1,175 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3407b6ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get data\n",
+    "import urllib\n",
+    "import json\n",
+    "url = 'https://data.nationalgrideso.com/api/3/action/datastore_search?resource_id=bbe2cc72-a6c6-46e6-8f4e-48b879467368&limit=400'\n",
+    "fileobj = urllib.request.urlopen(url)\n",
+    "d = json.loads(fileobj.read())\n",
+    "\n",
+    "print(d)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6a58b2c7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load data\n",
+    "import pandas as pd\n",
+    "results = d['result']['records']\n",
+    "\n",
+    "data_df = pd.DataFrame(results)\n",
+    "print(len(data_df))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb615160",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# plot data\n",
+    "\n",
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "fig = go.Figure(data=go.Scatter(x=data_df['gsp_lon'], y=data_df['gsp_lat'], mode='markers'))\n",
+    "\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ddfc9698",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# plot on static map\n",
+    "\n",
+    "import staticmaps\n",
+    "from staticmaps.marker import Marker\n",
+    "\n",
+    "\n",
+    "context = staticmaps.Context()\n",
+    "\n",
+    "# make bottom left and top right bounding box for map, happens to be Oxford and Norwich\n",
+    "bottom_left = staticmaps.create_latlng(50, -8)\n",
+    "top_right = staticmaps.create_latlng(59, 3)\n",
+    "\n",
+    "for i in range(len(data_df)-4):\n",
+    "    row = data_df.iloc[i]\n",
+    "    context.add_object(Marker(staticmaps.create_latlng(row.gnode_lat, row.gnode_lon), size=3))\n",
+    "    \n",
+    "    \n",
+    "# make clean map\n",
+    "m = context.make_clean_map_from_bounding_box(bottom_left=bottom_left, top_right=top_right, width=1000, height=1000)\n",
+    "\n",
+    "m.show()\n",
+    "m.save('GSP.png')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d5e08e2d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# plot using plotly\n",
+    "import numpy as np\n",
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "m = np.array(m)\n",
+    "print(m.shape)\n",
+    "\n",
+    "x = (data_df['gnode_lon'] +8) / 11 * m.shape[1]\n",
+    "y = (59-data_df['gnode_lat']) / 9 * m.shape[0]\n",
+    "\n",
+    "trace_map = go.Image(z=m)\n",
+    "\n",
+    "layout = go.Layout(\n",
+    "    paper_bgcolor='rgba(0,0,0,0)',\n",
+    "    plot_bgcolor='rgba(0,0,0,0)'\n",
+    ")\n",
+    "\n",
+    "fig = go.Figure(data=[trace_map, go.Scatter(x=x, y=y, mode='markers', text=data_df['gnode_name'],  \n",
+    "                                            marker=dict(color='LightSkyBlue'))],\n",
+    "               layout=layout)\n",
+    "\n",
+    "fig.update_yaxes(showticklabels=False)\n",
+    "fig.update_xaxes(showticklabels=False)\n",
+    "\n",
+    "\n",
+    "\n",
+    "fig.show()\n",
+    "\n",
+    "### This doesnt quite work, due to curitual of the world\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c4bffdb2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2548a7da",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ad608b5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9087e598",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/2021-09/2021-09-01/GSP solar data.ipynb

@@ -0,0 +1,177 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e5860ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get gsp metadata\n",
+    "import urllib\n",
+    "import json\n",
+    "import pandas as pd\n",
+    "\n",
+    "from pvlive_api import PVLive\n",
+    "from datetime import datetime, timedelta\n",
+    "import pytz\n",
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "# call ESO website\n",
+    "url = 'https://data.nationalgrideso.com/api/3/action/datastore_search?resource_id=bbe2cc72-a6c6-46e6-8f4e-48b879467368&limit=400'\n",
+    "fileobj = urllib.request.urlopen(url)\n",
+    "d = json.loads(fileobj.read())\n",
+    "\n",
+    "# make dataframe\n",
+    "results = d['result']['records']\n",
+    "metadata_df = pd.DataFrame(results)\n",
+    "\n",
+    "print(metadata_df)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2ab08c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# plot one day of data\n",
+    "# https://github.com/SheffieldSolar/PV_Live-API, use this repo\n",
+    "\n",
+    "pvl = PVLive()\n",
+    "\n",
+    "# test getting some data\n",
+    "start = datetime(2019,1,1,tzinfo=pytz.utc)\n",
+    "end = datetime(2019,1,2,tzinfo=pytz.utc)\n",
+    "\n",
+    "one_day_gsp_data_df = pvl.between(start, end, entity_type=\"gsp\", entity_id=0, extra_fields=\"\", dataframe=True)\n",
+    "\n",
+    "one_day_gsp_data_df = one_day_gsp_data_df.sort_values(by=['datetime_gmt'])\n",
+    "\n",
+    "fig = go.Figure(data=go.Scatter(x=one_gsp_data_df['datetime_gmt'], y=one_gsp_data_df['generation_mw']))\n",
+    "fig.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52f31238",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# plot one month of data\n",
+    "\n",
+    "pvl = PVLive()\n",
+    "\n",
+    "# test getting some data\n",
+    "start = datetime(2019,1,1,tzinfo=pytz.utc)\n",
+    "end = datetime(2019,2,1, tzinfo=pytz.utc)\n",
+    "\n",
+    "one_month_gsp_data_df = pvl.between(start, end, entity_type=\"gsp\", entity_id=0, extra_fields=\"\", dataframe=True)\n",
+    "\n",
+    "one_month_gsp_data_df = one_month_gsp_data_df.sort_values(by=['datetime_gmt'])\n",
+    "\n",
+    "fig = go.Figure(data=go.Scatter(x=one_month_gsp_data_df['datetime_gmt'], y=one_month_gsp_data_df['generation_mw']))\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4107dab0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# plot one month of data, for 10 sites\n",
+    "from datetime import datetime, timedelta\n",
+    "pvl = PVLive()\n",
+    "\n",
+    "# test getting some data\n",
+    "start = datetime(2019,1,1,tzinfo=pytz.utc)\n",
+    "end = datetime(2019,6,1, tzinfo=pytz.utc)\n",
+    "\n",
+    "N_gsp_ids = 20\n",
+    "\n",
+    "one_month_gsp_data_df = []\n",
+    "for i in range(0,20):\n",
+    "    start_chunk = start\n",
+    "    end_chunk = start_chunk + timedelta(days=30)\n",
+    "    while end_chunk < end:\n",
+    "        print(f'Getting data for id {i} from {start_chunk} to {end_chunk}')\n",
+    "        one_month_gsp_data_df.append(pvl.between(start=start_chunk, \n",
+    "                                                 end=end_chunk, \n",
+    "                                                 entity_type=\"gsp\", \n",
+    "                                                 entity_id=i, \n",
+    "                                                 extra_fields=\"\", \n",
+    "                                                 dataframe=True))\n",
+    "        \n",
+    "        start_chunk = start_chunk + timedelta(days=30)\n",
+    "        end_chunk = end_chunk + timedelta(days=30)\n",
+    "        \n",
+    "        \n",
+    "one_month_gsp_data_df = pd.concat(one_month_gsp_data_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3520f01e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "one_month_gsp_data_df = one_month_gsp_data_df.sort_values(by=['gsp_id','datetime_gmt'])\n",
+    "\n",
+    "fig = go.Figure()\n",
+    "for i in range(0,N_gsp_ids):\n",
+    "    temp_df = one_month_gsp_data_df[one_month_gsp_data_df['gsp_id'] == i]\n",
+    "    \n",
+    "    fig.add_trace(go.Scatter(x=temp_df['datetime_gmt'], \n",
+    "                                y=temp_df['generation_mw'], name=metadata_df.loc[i].gnode_name))\n",
+    "\n",
+    "fig.update_layout(title='GSP solar data')\n",
+    "fig.update_yaxes(title='MW')\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6744491f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(metadata_df.loc[1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "00e093dd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}