Add maping monitors.

deploy_boulia/05_Monitor_Mapping.ipynb

@@ -0,0 +1,354 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "707bb7bc-2991-4d25-8261-dede3fd111c3",
+   "metadata": {},
+   "source": [
+    "# Monitor progress of mapping tasks\n",
+    "\n",
+    "Run this notebook to visualise the progress of mapping tasks."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9d92f667-1d95-43e8-b47f-a928cdce5907",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Necessary imports\n",
+    "import os\n",
+    "os.environ['USE_PYGEOS'] = '0'\n",
+    "import pandas as pd\n",
+    "import geopandas as gpd\n",
+    "import time\n",
+    "import folium\n",
+    "import geemap.foliumap as geemap\n",
+    "import branca.colormap\n",
+    "from tqdm.notebook import tqdm\n",
+    "from datetime import datetime\n",
+    "\n",
+    "from db_utils import DB\n",
+    "from dotenv import load_dotenv"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5d30dd68",
+   "metadata": {},
+   "source": [
+    "## Load environment and project details\n",
+    "\n",
+    "As with the other notebooks, we load credentials and project details from a hidden ```.env``` file."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e57b7c05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load environment variables (including path to credentials) from '.env' file\n",
+    "env_file_path = \"../.env\"\n",
+    "\n",
+    "# Uncomment for alternative version for Windows (r\"\" indicates raw string)\n",
+    "#env_file_path = r\"C:/Users/User/floodmapper/.env\"\n",
+    "\n",
+    "assert load_dotenv(dotenv_path=env_file_path) == True, \"[ERR] Failed to load environment!\"\n",
+    "assert \"GOOGLE_APPLICATION_CREDENTIALS\" in os.environ, \"[ERR] Missing $GOOGLE_APPLICATION_CREDENTIAL!\"\n",
+    "assert \"GS_USER_PROJECT\" in os.environ, \"[ERR] Missing $GS_USER_PROJECT!\"\n",
+    "key_file_path = os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"]\n",
+    "assert os.path.exists(key_file_path), f\"[ERR] Google credential key file does not exist: \\n{key_file_path} \"\n",
+    "assert \"ML4FLOODS_BASE_DIR\" in os.environ, \"[ERR] Missing $ML4FLOODS_BASE_DIR!\"\n",
+    "base_path = os.environ[\"ML4FLOODS_BASE_DIR\"]\n",
+    "assert os.path.exists(base_path), f\"[ERR] Base path does not exist: \\n{base_path} \"\n",
+    "print(\"[INFO] Successfully loaded FloodMapper environment.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "21371a78",
+   "metadata": {},
+   "source": [
+    "## Query the progress of inference from database\n",
+    "\n",
+    "**Set the name of the session here and run all remaining cells in order.**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e1aebf3c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# EDIT THE NAME OF THE SESSION\n",
+    "session_name = \"boulia_test\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b404539e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Connect to the database (point to the .env file for credentials)\n",
+    "db_conn = DB(env_file_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cf803cdd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Query the geometry of the selected area\n",
+    "query = (f\"SELECT DISTINCT sp.patch_name, ST_AsText(gr.geometry) \"\n",
+    "         f\"FROM session_patches sp \"\n",
+    "         f\"INNER JOIN grid_loc gr \"\n",
+    "         f\"ON sp.patch_name = gr.patch_name \"\n",
+    "         f\"WHERE sp.session = %s ;\")\n",
+    "data = (session_name,)\n",
+    "grid_sel_df = db_conn.run_query(query, data, fetch=True)\n",
+    "print(f\"[INFO] Returned {len(grid_sel_df)} rows.\")\n",
+    "\n",
+    "# Format the results into a correct GeoDataFrame\n",
+    "grid_sel_df['geometry'] = gpd.GeoSeries.from_wkt(grid_sel_df['st_astext'])\n",
+    "grid_sel_df.drop(['st_astext'], axis=1, inplace = True)\n",
+    "grid_sel_gdf = gpd.GeoDataFrame(grid_sel_df, geometry='geometry', crs=\"EPSG:4326\")\n",
+    "print(grid_sel_gdf.head(3))\n",
+    "\n",
+    "# Create an outline of the map area\n",
+    "aoi_outline = grid_sel_gdf.geometry.unary_union\n",
+    "aoi_outline_gdf = gpd.GeoDataFrame(geometry=[aoi_outline], crs=\"EPSG:4326\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a753763a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fetch the session parameters from the database\n",
+    "query = (f\"SELECT flood_date_start, flood_date_end, \"\n",
+    "         f\"ref_date_start, ref_date_end, bucket_uri \"\n",
+    "         f\"FROM session_info \"\n",
+    "         f\"WHERE session = %s\")\n",
+    "data = (session_name,)\n",
+    "session_df = db_conn.run_query(query, data, fetch=True)\n",
+    "flood_start_date = session_df.iloc[0][\"flood_date_start\"]\n",
+    "flood_end_date = session_df.iloc[0][\"flood_date_end\"]\n",
+    "ref_start_date = session_df.iloc[0][\"ref_date_start\"]\n",
+    "ref_end_date = session_df.iloc[0][\"ref_date_end\"]\n",
+    "bucket_uri = session_df.iloc[0][\"bucket_uri\"]\n",
+    "\n",
+    "# Fetch the AoI grid patches from the database\n",
+    "query = (f\"SELECT DISTINCT patch_name \"\n",
+    "         f\"FROM session_patches \"\n",
+    "         f\"WHERE session = %s\")\n",
+    "data = (session_name,)\n",
+    "aois_df = db_conn.run_query(query, data, fetch=True)\n",
+    "num_patches = len(aois_df)\n",
+    "print(f\"[INFO] Found {num_patches} grid patches in map.\")\n",
+    "aois_list = aois_df.patch_name.to_list()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e97d59e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Query the inference status and geometry of each image\n",
+    "query = (f\"SELECT DISTINCT dl.image_id, dl.patch_name, inf.status, ST_AsText(gr.geometry) \"\n",
+    "         f\"FROM image_downloads dl \"\n",
+    "         f\"LEFT JOIN grid_loc gr \"\n",
+    "         f\"ON dl.patch_name = gr.patch_name \"\n",
+    "         f\"LEFT JOIN (SELECT * FROM inference WHERE mode='vect') AS inf \"\n",
+    "         f\"ON dl.image_id = inf.image_id \"\n",
+    "         f\"WHERE dl.patch_name IN %s \"\n",
+    "         f\"AND dl.status = 1 \"\n",
+    "         f\"AND ((dl.date >= %s \"\n",
+    "         f\"AND dl.date <= %s) \")\n",
+    "data = [tuple(aois_list), flood_start_date, flood_end_date]\n",
+    "if ref_start_date is not None and ref_end_date is not None:\n",
+    "    query += (f\"OR (dl.date >= %s \"\n",
+    "              f\"AND dl.date <= %s));\")\n",
+    "    data += [ref_start_date, ref_end_date]\n",
+    "else:\n",
+    "    query += (f\");\")\n",
+    "inf_df = db_conn.run_query(query, data, fetch = True)\n",
+    "num_rows = len(inf_df)\n",
+    "print(f\"[INFO] Entries for {num_rows} images in the DB.\")\n",
+    "\n",
+    "# Format the results into a correct GeoDataFrame\n",
+    "inf_df['geometry'] = gpd.GeoSeries.from_wkt(inf_df['st_astext'])\n",
+    "inf_df.drop(['st_astext'], axis=1, inplace = True)\n",
+    "inf_gdf = gpd.GeoDataFrame(inf_df, geometry='geometry', crs=\"EPSG:4326\")\n",
+    "inf_gdf = inf_gdf.fillna(0)\n",
+    "inf_gdf.head(3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8b754f2b",
+   "metadata": {},
+   "source": [
+    "## Parse the number of processed files in each grid patch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23f569d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Extract the patch polygons\n",
+    "geom = inf_gdf[[\"patch_name\", \"geometry\"]].drop_duplicates()\n",
+    "geom = geom.set_index(\"patch_name\")\n",
+    "\n",
+    "# Count the processed and unprocessed files\n",
+    "grid_inf_gdf = inf_gdf.loc[inf_gdf.status == 1]\n",
+    "processed = grid_inf_gdf.groupby(\"patch_name\").image_id.count()\n",
+    "grid_not_gdf = inf_gdf.loc[inf_gdf.status == 0]\n",
+    "unprocessed = grid_not_gdf.groupby(\"patch_name\").image_id.count()\n",
+    "\n",
+    "# Create a processed gdf\n",
+    "processed_df = pd.concat([processed, geom], axis = 1)\n",
+    "processed_df = processed_df.rename(columns={\"image_id\": \"count\"})\n",
+    "processed_gdf = gpd.GeoDataFrame(processed_df, geometry='geometry', crs=\"EPSG:4326\")\n",
+    "processed_gdf = processed_gdf.fillna(0)\n",
+    "\n",
+    "# Create an unprocessed gdf\n",
+    "unprocessed_df = pd.concat([unprocessed, geom], axis = 1)\n",
+    "unprocessed_df = unprocessed_df.rename(columns={\"image_id\": \"count\"})\n",
+    "unprocessed_gdf = gpd.GeoDataFrame(unprocessed_df, geometry='geometry', crs=\"EPSG:4326\")\n",
+    "unprocessed_gdf = unprocessed_gdf.fillna(0)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fa446f38",
+   "metadata": {},
+   "source": [
+    "## Plot the number of processed files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65a7b63e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define a style function to set the colours\n",
+    "#cm = branca.colormap.LinearColormap(\n",
+    "#    ['red', 'orange', 'yellow', 'cyan', 'blue', 'darkblue'],\n",
+    "cm = branca.colormap.linear.YlOrRd_07.scale(\n",
+    "    vmin=processed_gdf[\"count\"].min(), \n",
+    "    vmax=processed_gdf[\"count\"].max())\n",
+    "def style_fn(feature):\n",
+    "    return {\n",
+    "        'fillColor': cm(feature['properties']['count']),\n",
+    "        'color': cm(feature['properties']['count']),\n",
+    "        'weight': 0.5,\n",
+    "        \"fillOpacity\": 0.5\n",
+    "    }\n",
+    "\n",
+    "# Plot the patches colour-coded by number of downloads\n",
+    "m = aoi_outline_gdf.explore(color=\"black\", style_kwds={\"fillOpacity\": 0.0, \"weight\": 3}, \n",
+    "                            name=\"AoI Outline\", highlight=False)\n",
+    "folium.GeoJson(processed_gdf, \n",
+    "               style_function=style_fn,\n",
+    "               name=\"Processed Images\",\n",
+    "               tooltip=folium.features.GeoJsonTooltip([\"count\"]),\n",
+    "              ).add_to(m)\n",
+    "\n",
+    "# Add the colourmap, layer control and show\n",
+    "m.add_child(cm)\n",
+    "folium.LayerControl(collapsed=False).add_to(m)\n",
+    "m"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "809c4889",
+   "metadata": {},
+   "source": [
+    "## Plot the number of unprocessed files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d51a7fa3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define a style function to set the colours\n",
+    "#cm = branca.colormap.LinearColormap(\n",
+    "#    ['red', 'yellow', 'green'],\n",
+    "cm = branca.colormap.linear.YlOrRd_07.scale(\n",
+    "    vmin=unprocessed_gdf[\"count\"].min(), \n",
+    "    vmax=unprocessed_gdf[\"count\"].max())\n",
+    "def style_fn(feature):\n",
+    "    return {\n",
+    "        'fillColor': cm(feature['properties']['count']),\n",
+    "        'color': cm(feature['properties']['count']),\n",
+    "        'weight': 0.5,\n",
+    "        \"fillOpacity\": 0.5\n",
+    "    }\n",
+    "\n",
+    "# Plot the patches colour-coded by number of downloads\n",
+    "m = aoi_outline_gdf.explore(color=\"black\", style_kwds={\"fillOpacity\": 0.0, \"weight\": 3}, \n",
+    "                            name=\"AoI Outline\", highlight=False)\n",
+    "folium.GeoJson(unprocessed_gdf, \n",
+    "               style_function=style_fn,\n",
+    "               name=\"Unprocessed Images\",\n",
+    "               tooltip=folium.features.GeoJsonTooltip([\"count\"]),\n",
+    "              ).add_to(m)\n",
+    "\n",
+    "# Add the colourmap, layer control and show\n",
+    "m.add_child(cm)\n",
+    "folium.LayerControl(collapsed=False).add_to(m)\n",
+    "m"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "944750ad-bab7-422e-9714-e49522d2e029",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (floodmapper)",
+   "language": "python",
+   "name": "floodmapper"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}