diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 362f3b2a375..6668316a64e 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -329,6 +329,11 @@ VHR-10 .. autoclass:: VHR10 +Western USA Live Fuel Moisture +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: WesternUSALiveFuelMoisture + xView2 ^^^^^^ diff --git a/docs/api/non_geo_datasets.csv b/docs/api/non_geo_datasets.csv index 035c8cba9d7..4f7731e934f 100644 --- a/docs/api/non_geo_datasets.csv +++ b/docs/api/non_geo_datasets.csv @@ -33,5 +33,6 @@ Dataset,Task,Source,# Samples,# Classes,Size (px),Resolution (m),Bands `USAVars`_,R,NAIP Aerial,100K,-,-,4,"RGB, NIR" `Vaihingen`_,S,Aerial,33,6,"1,281--3,816",0.09,RGB `VHR-10`_,I,"Google Earth, Vaihingen",800,10,"358--1,728",0.08--2,RGB +`Western USA Live Fuel Moisture`_,R,"Landsat8, Sentinel-1",2615,-,-,-,- `xView2`_,CD,Maxar,"3,732",4,"1,024x1,024",0.8,RGB `ZueriCrop`_,"I, T",Sentinel-2,116K,48,24x24,10,MSI diff --git a/tests/data/western_usa_live_fuel_moisture/data.py b/tests/data/western_usa_live_fuel_moisture/data.py new file mode 100644 index 00000000000..d64c445ed41 --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/data.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import hashlib +import json +import os +import shutil + +NUM_SAMPLES = 3 + + +data_dir = "su_sar_moisture_content" + +LABELS = { + "type": "Feature", + "properties": { + "percent(t)": 132.6666667, + "site": "Blackstone", + "date": "6/30/15", + "slope(t)": 0.599961042, + "elevation(t)": 1522.0, + "canopy_height(t)": 0.0, + "forest_cover(t)": 130.0, + "silt(t)": 36.0, + "sand(t)": 38.0, + "clay(t)": 26.0, + "vv(t)": -12.80108143, + "vh(t)": -20.86413967, + "red(t)": 2007.5, + "green(t)": 1669.5, + "blue(t)": 1234.5, + "swir(t)": 3226.5, + "nir(t)": 2764.5, + "ndvi(t)": 0.158611467, + "ndwi(t)": -0.07713057, + "nirv(t)": 438.5596345, + "vv_red(t)": -0.006376628, + "vv_green(t)": -0.007667614, + "vv_blue(t)": -0.010369446, + "vv_swir(t)": -0.003967482, + "vv_nir(t)": -0.004630523, + "vv_ndvi(t)": -80.70716267, + "vv_ndwi(t)": 165.9663796, + "vv_nirv(t)": -0.029188919, + "vh_red(t)": -0.010393096, + "vh_green(t)": -0.012497238, + "vh_blue(t)": -0.016900883, + "vh_swir(t)": -0.006466493, + "vh_nir(t)": -0.007547166, + "vh_ndvi(t)": -131.5424422, + "vh_ndwi(t)": 270.5041557, + "vh_nirv(t)": -0.047574236, + "vh_vv(t)": -8.063058239, + "slope(t-1)": 0.599961042, + "elevation(t-1)": 1522.0, + "canopy_height(t-1)": 0.0, + "forest_cover(t-1)": 130.0, + "silt(t-1)": 36.0, + "sand(t-1)": 38.0, + "clay(t-1)": 26.0, + "vv(t-1)": -12.93716855, + "vh(t-1)": -20.92368901, + "red(t-1)": 1792.0, + "green(t-1)": 1490.0, + "blue(t-1)": 1102.5, + "swir(t-1)": 3047.0, + "nir(t-1)": 2574.0, + "ndvi(t-1)": 0.179116009, + "ndwi(t-1)": -0.084146807, + "nirv(t-1)": 461.0691997, + "vv_red(t-1)": -0.007219402, + "vv_green(t-1)": -0.008682663, + "vv_blue(t-1)": -0.011734393, + "vv_swir(t-1)": -0.004245871, + "vv_nir(t-1)": -0.005026095, + "vv_ndvi(t-1)": -72.22787422, + "vv_ndwi(t-1)": 153.7452097, + "vv_nirv(t-1)": -0.02805906, + "vh_red(t-1)": -0.011676166, + "vh_green(t-1)": -0.014042744, + "vh_blue(t-1)": -0.018978403, + "vh_swir(t-1)": -0.00686698, + "vh_nir(t-1)": -0.008128861, + "vh_ndvi(t-1)": -116.8164094, + "vh_ndwi(t-1)": 248.6569562, + "vh_nirv(t-1)": -0.0453808, + "vh_vv(t-1)": -7.986520458, + "slope(t-2)": 0.599961042, + "elevation(t-2)": 1522.0, + "canopy_height(t-2)": 0.0, + "forest_cover(t-2)": 130.0, + "silt(t-2)": 36.0, + "sand(t-2)": 38.0, + "clay(t-2)": 26.0, + "vv(t-2)": -13.07325567, + "vh(t-2)": -20.98323835, + "red(t-2)": 1721.5, + "green(t-2)": 1432.0, + "blue(t-2)": 1056.5, + "swir(t-2)": 2950.0, + "nir(t-2)": 2476.0, + "ndvi(t-2)": 0.179768568, + "ndwi(t-2)": -0.087357002, + "nirv(t-2)": 445.0984812, + "vv_red(t-2)": -0.007594107, + "vv_green(t-2)": -0.009129368, + "vv_blue(t-2)": -0.012374118, + "vv_swir(t-2)": -0.004431612, + "vv_nir(t-2)": -0.00527999, + "vv_ndvi(t-2)": -72.72270011, + "vv_ndwi(t-2)": 149.6532084, + "vv_nirv(t-2)": -0.029371603, + "vh_red(t-2)": -0.012188927, + "vh_green(t-2)": -0.014653099, + "vh_blue(t-2)": -0.019861087, + "vh_swir(t-2)": -0.007112962, + "vh_nir(t-2)": -0.008474652, + "vh_ndvi(t-2)": -116.7236217, + "vh_ndwi(t-2)": 240.2009889, + "vh_nirv(t-2)": -0.047142912, + "vh_vv(t-2)": -7.909982677, + "slope(t-3)": 0.599961042, + "elevation(t-3)": 1522.0, + "canopy_height(t-3)": 0.0, + "forest_cover(t-3)": 130.0, + "silt(t-3)": 36.0, + "sand(t-3)": 38.0, + "clay(t-3)": 26.0, + "vv(t-3)": -12.35794964, + "vh(t-3)": -20.25746909, + "red(t-3)": 1367.333333, + "green(t-3)": 1151.0, + "blue(t-3)": 827.3333333, + "swir(t-3)": 2349.333333, + "nir(t-3)": 2051.0, + "ndvi(t-3)": 0.216978329, + "ndwi(t-3)": -0.050717071, + "nirv(t-3)": 413.3885932, + "vv_red(t-3)": -0.009037993, + "vv_green(t-3)": -0.010736707, + "vv_blue(t-3)": -0.014937087, + "vv_swir(t-3)": -0.005260194, + "vv_nir(t-3)": -0.006025329, + "vv_ndvi(t-3)": -56.95476465, + "vv_ndwi(t-3)": 243.6644995, + "vv_nirv(t-3)": -0.029894269, + "vh_red(t-3)": -0.014815311, + "vh_green(t-3)": -0.017599886, + "vh_blue(t-3)": -0.024485257, + "vh_swir(t-3)": -0.008622646, + "vh_nir(t-3)": -0.009876874, + "vh_ndvi(t-3)": -93.36171601, + "vh_ndwi(t-3)": 399.4211186, + "vh_nirv(t-3)": -0.049003454, + "vh_vv(t-3)": -7.899519455, + }, + "geometry": {"type": "Point", "coordinates": [-115.8855556, 42.44111111]}, +} + +STAC = { + "assets": { + "documentation": { + "href": "../_common/documentation.pdf", + "type": "application/pdf", + }, + "labels": {"href": "labels.geojson", "type": "application/geo+json"}, + "training_features_descriptions": { + "href": "../_common/training_features_descriptions.csv", + "title": "Training Features Descriptions", + "type": "text/csv", + }, + }, + "bbox": [-115.8855556, 42.44111111, -115.8855556, 42.44111111], + "collection": "su_sar_moisture_content", + "geometry": {"coordinates": [-115.8855556, 42.44111111], "type": "Point"}, + "id": "su_sar_moisture_content_0001", + "links": [ + {"href": "../collection.json", "rel": "collection"}, + {"href": "../collection.json", "rel": "parent"}, + ], + "properties": { + "datetime": "2015-06-30T00:00:00Z", + "label:description": "", + "label:properties": ["percent(t)"], + "label:type": "vector", + }, + "stac_extensions": ["label"], + "stac_version": "1.0.0-beta.2", + "type": "Feature", +} + + +def create_file(path: str) -> None: + label_path = os.path.join(path, "labels.geojson") + with open(label_path, "w") as f: + json.dump(LABELS, f) + + stac_path = os.path.join(path, "stac.json") + with open(stac_path, "w") as f: + json.dump(STAC, f) + + +if __name__ == "__main__": + # Remove old data + if os.path.isdir(data_dir): + shutil.rmtree(data_dir) + + os.makedirs(os.path.join(os.getcwd(), data_dir)) + + for i in range(NUM_SAMPLES): + sample_dir = os.path.join(data_dir, data_dir + f"_{i}") + os.makedirs(sample_dir) + create_file(sample_dir) + + # Compress data + shutil.make_archive(data_dir, "gztar", ".", data_dir) + + # Compute checksums + with open(data_dir + ".tar.gz", "rb") as f: + md5 = hashlib.md5(f.read()).hexdigest() + print(f"{data_dir}.tar.gz: {md5}") diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content.tar.gz b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content.tar.gz new file mode 100644 index 00000000000..fe38d993d67 Binary files /dev/null and b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content.tar.gz differ diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_0/labels.geojson b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_0/labels.geojson new file mode 100644 index 00000000000..9492503feae --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_0/labels.geojson @@ -0,0 +1 @@ +{"type": "Feature", "properties": {"percent(t)": 132.6666667, "site": "Blackstone", "date": "6/30/15", "slope(t)": 0.599961042, "elevation(t)": 1522.0, "canopy_height(t)": 0.0, "forest_cover(t)": 130.0, "silt(t)": 36.0, "sand(t)": 38.0, "clay(t)": 26.0, "vv(t)": -12.80108143, "vh(t)": -20.86413967, "red(t)": 2007.5, "green(t)": 1669.5, "blue(t)": 1234.5, "swir(t)": 3226.5, "nir(t)": 2764.5, "ndvi(t)": 0.158611467, "ndwi(t)": -0.07713057, "nirv(t)": 438.5596345, "vv_red(t)": -0.006376628, "vv_green(t)": -0.007667614, "vv_blue(t)": -0.010369446, "vv_swir(t)": -0.003967482, "vv_nir(t)": -0.004630523, "vv_ndvi(t)": -80.70716267, "vv_ndwi(t)": 165.9663796, "vv_nirv(t)": -0.029188919, "vh_red(t)": -0.010393096, "vh_green(t)": -0.012497238, "vh_blue(t)": -0.016900883, "vh_swir(t)": -0.006466493, "vh_nir(t)": -0.007547166, "vh_ndvi(t)": -131.5424422, "vh_ndwi(t)": 270.5041557, "vh_nirv(t)": -0.047574236, "vh_vv(t)": -8.063058239, "slope(t-1)": 0.599961042, "elevation(t-1)": 1522.0, "canopy_height(t-1)": 0.0, "forest_cover(t-1)": 130.0, "silt(t-1)": 36.0, "sand(t-1)": 38.0, "clay(t-1)": 26.0, "vv(t-1)": -12.93716855, "vh(t-1)": -20.92368901, "red(t-1)": 1792.0, "green(t-1)": 1490.0, "blue(t-1)": 1102.5, "swir(t-1)": 3047.0, "nir(t-1)": 2574.0, "ndvi(t-1)": 0.179116009, "ndwi(t-1)": -0.084146807, "nirv(t-1)": 461.0691997, "vv_red(t-1)": -0.007219402, "vv_green(t-1)": -0.008682663, "vv_blue(t-1)": -0.011734393, "vv_swir(t-1)": -0.004245871, "vv_nir(t-1)": -0.005026095, "vv_ndvi(t-1)": -72.22787422, "vv_ndwi(t-1)": 153.7452097, "vv_nirv(t-1)": -0.02805906, "vh_red(t-1)": -0.011676166, "vh_green(t-1)": -0.014042744, "vh_blue(t-1)": -0.018978403, "vh_swir(t-1)": -0.00686698, "vh_nir(t-1)": -0.008128861, "vh_ndvi(t-1)": -116.8164094, "vh_ndwi(t-1)": 248.6569562, "vh_nirv(t-1)": -0.0453808, "vh_vv(t-1)": -7.986520458, "slope(t-2)": 0.599961042, "elevation(t-2)": 1522.0, "canopy_height(t-2)": 0.0, "forest_cover(t-2)": 130.0, "silt(t-2)": 36.0, "sand(t-2)": 38.0, "clay(t-2)": 26.0, "vv(t-2)": -13.07325567, "vh(t-2)": -20.98323835, "red(t-2)": 1721.5, "green(t-2)": 1432.0, "blue(t-2)": 1056.5, "swir(t-2)": 2950.0, "nir(t-2)": 2476.0, "ndvi(t-2)": 0.179768568, "ndwi(t-2)": -0.087357002, "nirv(t-2)": 445.0984812, "vv_red(t-2)": -0.007594107, "vv_green(t-2)": -0.009129368, "vv_blue(t-2)": -0.012374118, "vv_swir(t-2)": -0.004431612, "vv_nir(t-2)": -0.00527999, "vv_ndvi(t-2)": -72.72270011, "vv_ndwi(t-2)": 149.6532084, "vv_nirv(t-2)": -0.029371603, "vh_red(t-2)": -0.012188927, "vh_green(t-2)": -0.014653099, "vh_blue(t-2)": -0.019861087, "vh_swir(t-2)": -0.007112962, "vh_nir(t-2)": -0.008474652, "vh_ndvi(t-2)": -116.7236217, "vh_ndwi(t-2)": 240.2009889, "vh_nirv(t-2)": -0.047142912, "vh_vv(t-2)": -7.909982677, "slope(t-3)": 0.599961042, "elevation(t-3)": 1522.0, "canopy_height(t-3)": 0.0, "forest_cover(t-3)": 130.0, "silt(t-3)": 36.0, "sand(t-3)": 38.0, "clay(t-3)": 26.0, "vv(t-3)": -12.35794964, "vh(t-3)": -20.25746909, "red(t-3)": 1367.333333, "green(t-3)": 1151.0, "blue(t-3)": 827.3333333, "swir(t-3)": 2349.333333, "nir(t-3)": 2051.0, "ndvi(t-3)": 0.216978329, "ndwi(t-3)": -0.050717071, "nirv(t-3)": 413.3885932, "vv_red(t-3)": -0.009037993, "vv_green(t-3)": -0.010736707, "vv_blue(t-3)": -0.014937087, "vv_swir(t-3)": -0.005260194, "vv_nir(t-3)": -0.006025329, "vv_ndvi(t-3)": -56.95476465, "vv_ndwi(t-3)": 243.6644995, "vv_nirv(t-3)": -0.029894269, "vh_red(t-3)": -0.014815311, "vh_green(t-3)": -0.017599886, "vh_blue(t-3)": -0.024485257, "vh_swir(t-3)": -0.008622646, "vh_nir(t-3)": -0.009876874, "vh_ndvi(t-3)": -93.36171601, "vh_ndwi(t-3)": 399.4211186, "vh_nirv(t-3)": -0.049003454, "vh_vv(t-3)": -7.899519455}, "geometry": {"type": "Point", "coordinates": [-115.8855556, 42.44111111]}} \ No newline at end of file diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_0/stac.json b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_0/stac.json new file mode 100644 index 00000000000..469f98574d9 --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_0/stac.json @@ -0,0 +1 @@ +{"assets": {"documentation": {"href": "../_common/documentation.pdf", "type": "application/pdf"}, "labels": {"href": "labels.geojson", "type": "application/geo+json"}, "training_features_descriptions": {"href": "../_common/training_features_descriptions.csv", "title": "Training Features Descriptions", "type": "text/csv"}}, "bbox": [-115.8855556, 42.44111111, -115.8855556, 42.44111111], "collection": "su_sar_moisture_content", "geometry": {"coordinates": [-115.8855556, 42.44111111], "type": "Point"}, "id": "su_sar_moisture_content_0001", "links": [{"href": "../collection.json", "rel": "collection"}, {"href": "../collection.json", "rel": "parent"}], "properties": {"datetime": "2015-06-30T00:00:00Z", "label:description": "", "label:properties": ["percent(t)"], "label:type": "vector"}, "stac_extensions": ["label"], "stac_version": "1.0.0-beta.2", "type": "Feature"} \ No newline at end of file diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_1/labels.geojson b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_1/labels.geojson new file mode 100644 index 00000000000..9492503feae --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_1/labels.geojson @@ -0,0 +1 @@ +{"type": "Feature", "properties": {"percent(t)": 132.6666667, "site": "Blackstone", "date": "6/30/15", "slope(t)": 0.599961042, "elevation(t)": 1522.0, "canopy_height(t)": 0.0, "forest_cover(t)": 130.0, "silt(t)": 36.0, "sand(t)": 38.0, "clay(t)": 26.0, "vv(t)": -12.80108143, "vh(t)": -20.86413967, "red(t)": 2007.5, "green(t)": 1669.5, "blue(t)": 1234.5, "swir(t)": 3226.5, "nir(t)": 2764.5, "ndvi(t)": 0.158611467, "ndwi(t)": -0.07713057, "nirv(t)": 438.5596345, "vv_red(t)": -0.006376628, "vv_green(t)": -0.007667614, "vv_blue(t)": -0.010369446, "vv_swir(t)": -0.003967482, "vv_nir(t)": -0.004630523, "vv_ndvi(t)": -80.70716267, "vv_ndwi(t)": 165.9663796, "vv_nirv(t)": -0.029188919, "vh_red(t)": -0.010393096, "vh_green(t)": -0.012497238, "vh_blue(t)": -0.016900883, "vh_swir(t)": -0.006466493, "vh_nir(t)": -0.007547166, "vh_ndvi(t)": -131.5424422, "vh_ndwi(t)": 270.5041557, "vh_nirv(t)": -0.047574236, "vh_vv(t)": -8.063058239, "slope(t-1)": 0.599961042, "elevation(t-1)": 1522.0, "canopy_height(t-1)": 0.0, "forest_cover(t-1)": 130.0, "silt(t-1)": 36.0, "sand(t-1)": 38.0, "clay(t-1)": 26.0, "vv(t-1)": -12.93716855, "vh(t-1)": -20.92368901, "red(t-1)": 1792.0, "green(t-1)": 1490.0, "blue(t-1)": 1102.5, "swir(t-1)": 3047.0, "nir(t-1)": 2574.0, "ndvi(t-1)": 0.179116009, "ndwi(t-1)": -0.084146807, "nirv(t-1)": 461.0691997, "vv_red(t-1)": -0.007219402, "vv_green(t-1)": -0.008682663, "vv_blue(t-1)": -0.011734393, "vv_swir(t-1)": -0.004245871, "vv_nir(t-1)": -0.005026095, "vv_ndvi(t-1)": -72.22787422, "vv_ndwi(t-1)": 153.7452097, "vv_nirv(t-1)": -0.02805906, "vh_red(t-1)": -0.011676166, "vh_green(t-1)": -0.014042744, "vh_blue(t-1)": -0.018978403, "vh_swir(t-1)": -0.00686698, "vh_nir(t-1)": -0.008128861, "vh_ndvi(t-1)": -116.8164094, "vh_ndwi(t-1)": 248.6569562, "vh_nirv(t-1)": -0.0453808, "vh_vv(t-1)": -7.986520458, "slope(t-2)": 0.599961042, "elevation(t-2)": 1522.0, "canopy_height(t-2)": 0.0, "forest_cover(t-2)": 130.0, "silt(t-2)": 36.0, "sand(t-2)": 38.0, "clay(t-2)": 26.0, "vv(t-2)": -13.07325567, "vh(t-2)": -20.98323835, "red(t-2)": 1721.5, "green(t-2)": 1432.0, "blue(t-2)": 1056.5, "swir(t-2)": 2950.0, "nir(t-2)": 2476.0, "ndvi(t-2)": 0.179768568, "ndwi(t-2)": -0.087357002, "nirv(t-2)": 445.0984812, "vv_red(t-2)": -0.007594107, "vv_green(t-2)": -0.009129368, "vv_blue(t-2)": -0.012374118, "vv_swir(t-2)": -0.004431612, "vv_nir(t-2)": -0.00527999, "vv_ndvi(t-2)": -72.72270011, "vv_ndwi(t-2)": 149.6532084, "vv_nirv(t-2)": -0.029371603, "vh_red(t-2)": -0.012188927, "vh_green(t-2)": -0.014653099, "vh_blue(t-2)": -0.019861087, "vh_swir(t-2)": -0.007112962, "vh_nir(t-2)": -0.008474652, "vh_ndvi(t-2)": -116.7236217, "vh_ndwi(t-2)": 240.2009889, "vh_nirv(t-2)": -0.047142912, "vh_vv(t-2)": -7.909982677, "slope(t-3)": 0.599961042, "elevation(t-3)": 1522.0, "canopy_height(t-3)": 0.0, "forest_cover(t-3)": 130.0, "silt(t-3)": 36.0, "sand(t-3)": 38.0, "clay(t-3)": 26.0, "vv(t-3)": -12.35794964, "vh(t-3)": -20.25746909, "red(t-3)": 1367.333333, "green(t-3)": 1151.0, "blue(t-3)": 827.3333333, "swir(t-3)": 2349.333333, "nir(t-3)": 2051.0, "ndvi(t-3)": 0.216978329, "ndwi(t-3)": -0.050717071, "nirv(t-3)": 413.3885932, "vv_red(t-3)": -0.009037993, "vv_green(t-3)": -0.010736707, "vv_blue(t-3)": -0.014937087, "vv_swir(t-3)": -0.005260194, "vv_nir(t-3)": -0.006025329, "vv_ndvi(t-3)": -56.95476465, "vv_ndwi(t-3)": 243.6644995, "vv_nirv(t-3)": -0.029894269, "vh_red(t-3)": -0.014815311, "vh_green(t-3)": -0.017599886, "vh_blue(t-3)": -0.024485257, "vh_swir(t-3)": -0.008622646, "vh_nir(t-3)": -0.009876874, "vh_ndvi(t-3)": -93.36171601, "vh_ndwi(t-3)": 399.4211186, "vh_nirv(t-3)": -0.049003454, "vh_vv(t-3)": -7.899519455}, "geometry": {"type": "Point", "coordinates": [-115.8855556, 42.44111111]}} \ No newline at end of file diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_1/stac.json b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_1/stac.json new file mode 100644 index 00000000000..469f98574d9 --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_1/stac.json @@ -0,0 +1 @@ +{"assets": {"documentation": {"href": "../_common/documentation.pdf", "type": "application/pdf"}, "labels": {"href": "labels.geojson", "type": "application/geo+json"}, "training_features_descriptions": {"href": "../_common/training_features_descriptions.csv", "title": "Training Features Descriptions", "type": "text/csv"}}, "bbox": [-115.8855556, 42.44111111, -115.8855556, 42.44111111], "collection": "su_sar_moisture_content", "geometry": {"coordinates": [-115.8855556, 42.44111111], "type": "Point"}, "id": "su_sar_moisture_content_0001", "links": [{"href": "../collection.json", "rel": "collection"}, {"href": "../collection.json", "rel": "parent"}], "properties": {"datetime": "2015-06-30T00:00:00Z", "label:description": "", "label:properties": ["percent(t)"], "label:type": "vector"}, "stac_extensions": ["label"], "stac_version": "1.0.0-beta.2", "type": "Feature"} \ No newline at end of file diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_2/labels.geojson b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_2/labels.geojson new file mode 100644 index 00000000000..9492503feae --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_2/labels.geojson @@ -0,0 +1 @@ +{"type": "Feature", "properties": {"percent(t)": 132.6666667, "site": "Blackstone", "date": "6/30/15", "slope(t)": 0.599961042, "elevation(t)": 1522.0, "canopy_height(t)": 0.0, "forest_cover(t)": 130.0, "silt(t)": 36.0, "sand(t)": 38.0, "clay(t)": 26.0, "vv(t)": -12.80108143, "vh(t)": -20.86413967, "red(t)": 2007.5, "green(t)": 1669.5, "blue(t)": 1234.5, "swir(t)": 3226.5, "nir(t)": 2764.5, "ndvi(t)": 0.158611467, "ndwi(t)": -0.07713057, "nirv(t)": 438.5596345, "vv_red(t)": -0.006376628, "vv_green(t)": -0.007667614, "vv_blue(t)": -0.010369446, "vv_swir(t)": -0.003967482, "vv_nir(t)": -0.004630523, "vv_ndvi(t)": -80.70716267, "vv_ndwi(t)": 165.9663796, "vv_nirv(t)": -0.029188919, "vh_red(t)": -0.010393096, "vh_green(t)": -0.012497238, "vh_blue(t)": -0.016900883, "vh_swir(t)": -0.006466493, "vh_nir(t)": -0.007547166, "vh_ndvi(t)": -131.5424422, "vh_ndwi(t)": 270.5041557, "vh_nirv(t)": -0.047574236, "vh_vv(t)": -8.063058239, "slope(t-1)": 0.599961042, "elevation(t-1)": 1522.0, "canopy_height(t-1)": 0.0, "forest_cover(t-1)": 130.0, "silt(t-1)": 36.0, "sand(t-1)": 38.0, "clay(t-1)": 26.0, "vv(t-1)": -12.93716855, "vh(t-1)": -20.92368901, "red(t-1)": 1792.0, "green(t-1)": 1490.0, "blue(t-1)": 1102.5, "swir(t-1)": 3047.0, "nir(t-1)": 2574.0, "ndvi(t-1)": 0.179116009, "ndwi(t-1)": -0.084146807, "nirv(t-1)": 461.0691997, "vv_red(t-1)": -0.007219402, "vv_green(t-1)": -0.008682663, "vv_blue(t-1)": -0.011734393, "vv_swir(t-1)": -0.004245871, "vv_nir(t-1)": -0.005026095, "vv_ndvi(t-1)": -72.22787422, "vv_ndwi(t-1)": 153.7452097, "vv_nirv(t-1)": -0.02805906, "vh_red(t-1)": -0.011676166, "vh_green(t-1)": -0.014042744, "vh_blue(t-1)": -0.018978403, "vh_swir(t-1)": -0.00686698, "vh_nir(t-1)": -0.008128861, "vh_ndvi(t-1)": -116.8164094, "vh_ndwi(t-1)": 248.6569562, "vh_nirv(t-1)": -0.0453808, "vh_vv(t-1)": -7.986520458, "slope(t-2)": 0.599961042, "elevation(t-2)": 1522.0, "canopy_height(t-2)": 0.0, "forest_cover(t-2)": 130.0, "silt(t-2)": 36.0, "sand(t-2)": 38.0, "clay(t-2)": 26.0, "vv(t-2)": -13.07325567, "vh(t-2)": -20.98323835, "red(t-2)": 1721.5, "green(t-2)": 1432.0, "blue(t-2)": 1056.5, "swir(t-2)": 2950.0, "nir(t-2)": 2476.0, "ndvi(t-2)": 0.179768568, "ndwi(t-2)": -0.087357002, "nirv(t-2)": 445.0984812, "vv_red(t-2)": -0.007594107, "vv_green(t-2)": -0.009129368, "vv_blue(t-2)": -0.012374118, "vv_swir(t-2)": -0.004431612, "vv_nir(t-2)": -0.00527999, "vv_ndvi(t-2)": -72.72270011, "vv_ndwi(t-2)": 149.6532084, "vv_nirv(t-2)": -0.029371603, "vh_red(t-2)": -0.012188927, "vh_green(t-2)": -0.014653099, "vh_blue(t-2)": -0.019861087, "vh_swir(t-2)": -0.007112962, "vh_nir(t-2)": -0.008474652, "vh_ndvi(t-2)": -116.7236217, "vh_ndwi(t-2)": 240.2009889, "vh_nirv(t-2)": -0.047142912, "vh_vv(t-2)": -7.909982677, "slope(t-3)": 0.599961042, "elevation(t-3)": 1522.0, "canopy_height(t-3)": 0.0, "forest_cover(t-3)": 130.0, "silt(t-3)": 36.0, "sand(t-3)": 38.0, "clay(t-3)": 26.0, "vv(t-3)": -12.35794964, "vh(t-3)": -20.25746909, "red(t-3)": 1367.333333, "green(t-3)": 1151.0, "blue(t-3)": 827.3333333, "swir(t-3)": 2349.333333, "nir(t-3)": 2051.0, "ndvi(t-3)": 0.216978329, "ndwi(t-3)": -0.050717071, "nirv(t-3)": 413.3885932, "vv_red(t-3)": -0.009037993, "vv_green(t-3)": -0.010736707, "vv_blue(t-3)": -0.014937087, "vv_swir(t-3)": -0.005260194, "vv_nir(t-3)": -0.006025329, "vv_ndvi(t-3)": -56.95476465, "vv_ndwi(t-3)": 243.6644995, "vv_nirv(t-3)": -0.029894269, "vh_red(t-3)": -0.014815311, "vh_green(t-3)": -0.017599886, "vh_blue(t-3)": -0.024485257, "vh_swir(t-3)": -0.008622646, "vh_nir(t-3)": -0.009876874, "vh_ndvi(t-3)": -93.36171601, "vh_ndwi(t-3)": 399.4211186, "vh_nirv(t-3)": -0.049003454, "vh_vv(t-3)": -7.899519455}, "geometry": {"type": "Point", "coordinates": [-115.8855556, 42.44111111]}} \ No newline at end of file diff --git a/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_2/stac.json b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_2/stac.json new file mode 100644 index 00000000000..469f98574d9 --- /dev/null +++ b/tests/data/western_usa_live_fuel_moisture/su_sar_moisture_content/su_sar_moisture_content_2/stac.json @@ -0,0 +1 @@ +{"assets": {"documentation": {"href": "../_common/documentation.pdf", "type": "application/pdf"}, "labels": {"href": "labels.geojson", "type": "application/geo+json"}, "training_features_descriptions": {"href": "../_common/training_features_descriptions.csv", "title": "Training Features Descriptions", "type": "text/csv"}}, "bbox": [-115.8855556, 42.44111111, -115.8855556, 42.44111111], "collection": "su_sar_moisture_content", "geometry": {"coordinates": [-115.8855556, 42.44111111], "type": "Point"}, "id": "su_sar_moisture_content_0001", "links": [{"href": "../collection.json", "rel": "collection"}, {"href": "../collection.json", "rel": "parent"}], "properties": {"datetime": "2015-06-30T00:00:00Z", "label:description": "", "label:properties": ["percent(t)"], "label:type": "vector"}, "stac_extensions": ["label"], "stac_version": "1.0.0-beta.2", "type": "Feature"} \ No newline at end of file diff --git a/tests/datasets/test_western_usa_live_fuel_moisture.py b/tests/datasets/test_western_usa_live_fuel_moisture.py new file mode 100644 index 00000000000..ad79ca7937c --- /dev/null +++ b/tests/datasets/test_western_usa_live_fuel_moisture.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import builtins +import os +import shutil +from pathlib import Path +from typing import Any + +import pytest +import torch +import torch.nn as nn +from _pytest.fixtures import SubRequest +from _pytest.monkeypatch import MonkeyPatch + +from torchgeo.datasets import WesternUSALiveFuelMoisture + + +class Collection: + def download(self, output_dir: str, **kwargs: str) -> None: + tarball_path = os.path.join( + "tests", + "data", + "western_usa_live_fuel_moisture", + "su_sar_moisture_content.tar.gz", + ) + shutil.copy(tarball_path, output_dir) + + +def fetch(collection_id: str, **kwargs: str) -> Collection: + return Collection() + + +class TestWesternUSALiveFuelMoisture: + @pytest.fixture + def dataset( + self, monkeypatch: MonkeyPatch, tmp_path: Path + ) -> WesternUSALiveFuelMoisture: + radiant_mlhub = pytest.importorskip("radiant_mlhub", minversion="0.2.1") + monkeypatch.setattr(radiant_mlhub.Collection, "fetch", fetch) + md5 = "ecbc9269dd27c4efe7aa887960054351" + monkeypatch.setattr(WesternUSALiveFuelMoisture, "md5", md5) + root = str(tmp_path) + transforms = nn.Identity() + return WesternUSALiveFuelMoisture( + root, transforms=transforms, download=True, api_key="", checksum=True + ) + + @pytest.mark.parametrize("index", [0, 1, 2]) + def test_getitem(self, dataset: WesternUSALiveFuelMoisture, index: int) -> None: + x = dataset[index] + assert isinstance(x, dict) + assert isinstance(x["input"], torch.Tensor) + assert isinstance(x["label"], torch.Tensor) + + def test_len(self, dataset: WesternUSALiveFuelMoisture) -> None: + assert len(dataset) == 3 + + def test_already_downloaded(self, tmp_path: Path) -> None: + pathname = os.path.join( + "tests", + "data", + "western_usa_live_fuel_moisture", + "su_sar_moisture_content.tar.gz", + ) + root = str(tmp_path) + shutil.copy(pathname, root) + WesternUSALiveFuelMoisture(root) + + def test_not_downloaded(self, tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="Dataset not found in"): + WesternUSALiveFuelMoisture(str(tmp_path)) + + def test_invalid_features(self, dataset: WesternUSALiveFuelMoisture) -> None: + with pytest.raises(AssertionError, match="Invalid input variable name."): + WesternUSALiveFuelMoisture(dataset.root, input_features=["foo"]) + + @pytest.fixture(params=["pandas"]) + def mock_missing_module(self, monkeypatch: MonkeyPatch, request: SubRequest) -> str: + import_orig = builtins.__import__ + package = str(request.param) + + def mocked_import(name: str, *args: Any, **kwargs: Any) -> Any: + if name == package: + raise ImportError() + return import_orig(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", mocked_import) + return package + + def test_mock_missing_module( + self, dataset: WesternUSALiveFuelMoisture, mock_missing_module: str + ) -> None: + package = mock_missing_module + if package == "pandas": + with pytest.raises( + ImportError, + match=f"{package} is not installed and is required to use this dataset", + ): + WesternUSALiveFuelMoisture(dataset.root) diff --git a/torchgeo/datasets/__init__.py b/torchgeo/datasets/__init__.py index 2eff64d8696..ae25e0795ce 100644 --- a/torchgeo/datasets/__init__.py +++ b/torchgeo/datasets/__init__.py @@ -111,6 +111,7 @@ ) from .vaihingen import Vaihingen2D from .vhr10 import VHR10 +from .western_usa_live_fuel_moisture import WesternUSALiveFuelMoisture from .xview import XView2 from .zuericrop import ZueriCrop @@ -203,6 +204,7 @@ "USAVars", "Vaihingen2D", "VHR10", + "WesternUSALiveFuelMoisture", "XView2", "ZueriCrop", # Base classes diff --git a/torchgeo/datasets/western_usa_live_fuel_moisture.py b/torchgeo/datasets/western_usa_live_fuel_moisture.py new file mode 100644 index 00000000000..63fd8fbeccc --- /dev/null +++ b/torchgeo/datasets/western_usa_live_fuel_moisture.py @@ -0,0 +1,354 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Western USA Live Fuel Moisture Dataset.""" + +import glob +import json +import os +from typing import Any, Callable, Optional + +import torch +from torch import Tensor + +from .geo import NonGeoDataset +from .utils import download_radiant_mlhub_collection, extract_archive + + +class WesternUSALiveFuelMoisture(NonGeoDataset): + """Western USA Live Fuel Moisture Dataset. + + This tabular style dataset contains fuel moisture + (mass of water in vegetation) and remotely sensed variables + in the western United States. It contains 2615 datapoints and 138 + variables. For more details see the + `dataset page `_. + + Dataset Format: + + * .geojson file for each datapoint + + Dataset Features: + + * 138 remote sensing derived variables, some with a time dependency + * 2615 datapoints with regression target of predicting fuel moisture + + If you use this dataset in your research, please cite the following paper: + + * https://doi.org/10.1016/j.rse.2020.111797 + + .. note:: + + This dataset requires the following additional library to be installed: + + * `radiant-mlhub `_ to download the + imagery and labels from the Radiant Earth MLHub + + .. versionadded:: 0.5 + """ + + collection_id = "su_sar_moisture_content" + + md5 = "a6c0721f06a3a0110b7d1243b18614f0" + + label_name = "percent(t)" + + all_variable_names = [ + # "date", + "slope(t)", + "elevation(t)", + "canopy_height(t)", + "forest_cover(t)", + "silt(t)", + "sand(t)", + "clay(t)", + "vv(t)", + "vh(t)", + "red(t)", + "green(t)", + "blue(t)", + "swir(t)", + "nir(t)", + "ndvi(t)", + "ndwi(t)", + "nirv(t)", + "vv_red(t)", + "vv_green(t)", + "vv_blue(t)", + "vv_swir(t)", + "vv_nir(t)", + "vv_ndvi(t)", + "vv_ndwi(t)", + "vv_nirv(t)", + "vh_red(t)", + "vh_green(t)", + "vh_blue(t)", + "vh_swir(t)", + "vh_nir(t)", + "vh_ndvi(t)", + "vh_ndwi(t)", + "vh_nirv(t)", + "vh_vv(t)", + "slope(t-1)", + "elevation(t-1)", + "canopy_height(t-1)", + "forest_cover(t-1)", + "silt(t-1)", + "sand(t-1)", + "clay(t-1)", + "vv(t-1)", + "vh(t-1)", + "red(t-1)", + "green(t-1)", + "blue(t-1)", + "swir(t-1)", + "nir(t-1)", + "ndvi(t-1)", + "ndwi(t-1)", + "nirv(t-1)", + "vv_red(t-1)", + "vv_green(t-1)", + "vv_blue(t-1)", + "vv_swir(t-1)", + "vv_nir(t-1)", + "vv_ndvi(t-1)", + "vv_ndwi(t-1)", + "vv_nirv(t-1)", + "vh_red(t-1)", + "vh_green(t-1)", + "vh_blue(t-1)", + "vh_swir(t-1)", + "vh_nir(t-1)", + "vh_ndvi(t-1)", + "vh_ndwi(t-1)", + "vh_nirv(t-1)", + "vh_vv(t-1)", + "slope(t-2)", + "elevation(t-2)", + "canopy_height(t-2)", + "forest_cover(t-2)", + "silt(t-2)", + "sand(t-2)", + "clay(t-2)", + "vv(t-2)", + "vh(t-2)", + "red(t-2)", + "green(t-2)", + "blue(t-2)", + "swir(t-2)", + "nir(t-2)", + "ndvi(t-2)", + "ndwi(t-2)", + "nirv(t-2)", + "vv_red(t-2)", + "vv_green(t-2)", + "vv_blue(t-2)", + "vv_swir(t-2)", + "vv_nir(t-2)", + "vv_ndvi(t-2)", + "vv_ndwi(t-2)", + "vv_nirv(t-2)", + "vh_red(t-2)", + "vh_green(t-2)", + "vh_blue(t-2)", + "vh_swir(t-2)", + "vh_nir(t-2)", + "vh_ndvi(t-2)", + "vh_ndwi(t-2)", + "vh_nirv(t-2)", + "vh_vv(t-2)", + "slope(t-3)", + "elevation(t-3)", + "canopy_height(t-3)", + "forest_cover(t-3)", + "silt(t-3)", + "sand(t-3)", + "clay(t-3)", + "vv(t-3)", + "vh(t-3)", + "red(t-3)", + "green(t-3)", + "blue(t-3)", + "swir(t-3)", + "nir(t-3)", + "ndvi(t-3)", + "ndwi(t-3)", + "nirv(t-3)", + "vv_red(t-3)", + "vv_green(t-3)", + "vv_blue(t-3)", + "vv_swir(t-3)", + "vv_nir(t-3)", + "vv_ndvi(t-3)", + "vv_ndwi(t-3)", + "vv_nirv(t-3)", + "vh_red(t-3)", + "vh_green(t-3)", + "vh_blue(t-3)", + "vh_swir(t-3)", + "vh_nir(t-3)", + "vh_ndvi(t-3)", + "vh_ndwi(t-3)", + "vh_nirv(t-3)", + "vh_vv(t-3)", + "lat", + "lon", + ] + + def __init__( + self, + root: str = "data", + input_features: list[str] = all_variable_names, + transforms: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None, + download: bool = False, + api_key: Optional[str] = None, + checksum: bool = False, + ) -> None: + """Initialize a new Western USA Live Fuel Moisture Dataset. + + Args: + root: root directory where dataset can be found + input_features: which input features to include + transforms: a function/transform that takes input sample and its target as + entry and returns a transformed version + download: if True, download dataset and store it in the root directory + api_key: a RadiantEarth MLHub API key to use for downloading the dataset + checksum: if True, check the MD5 of the downloaded files (may be slow) + + Raises: + AssertionError: if ``input_features`` contains invalid variable names + ImportError: if pandas is not installed + RuntimeError: if ``download=False`` but dataset is missing or checksum fails + """ + super().__init__() + + self.root = root + self.transforms = transforms + self.checksum = checksum + self.download = download + self.api_key = api_key + + self._verify() + + try: + import pandas as pd # noqa: F401 + except ImportError: + raise ImportError( + "pandas is not installed and is required to use this dataset" + ) + + assert all( + input in self.all_variable_names for input in input_features + ), "Invalid input variable name." + self.input_features = input_features + + self.collection = self._retrieve_collection() + + self.dataframe = self._load_data() + + def _retrieve_collection(self) -> list[str]: + """Retrieve dataset collection that maps samples to paths. + + Returns: + list of sample paths + """ + return glob.glob( + os.path.join(self.root, self.collection_id, "**", "labels.geojson") + ) + + def __len__(self) -> int: + """Return the number of data points in the dataset. + + Returns: + length of the dataset + """ + return len(self.dataframe) + + def __getitem__(self, index: int) -> dict[str, Any]: + """Return an index within the dataset. + + Args: + index: index to return + + Returns: + input features and target at that index + """ + data = self.dataframe.iloc[index, :] + + sample: dict[str, Tensor] = { + "input": torch.tensor(data.drop([self.label_name]), dtype=torch.float32), + "label": torch.tensor(data[self.label_name], dtype=torch.float32), + } + + if self.transforms is not None: + sample = self.transforms(sample) + + return sample + + def _load_data(self) -> "pd.DataFrame": # type: ignore[name-defined] # noqa: F821 + """Load data from individual files into pandas dataframe. + + Returns: + the features and label + """ + import pandas as pd + + data_rows = [] + for path in self.collection: + with open(path) as f: + content = json.load(f) + data_dict = content["properties"] + data_dict["lat"] = content["geometry"]["coordinates"][0] + data_dict["lon"] = content["geometry"]["coordinates"][1] + data_rows.append(data_dict) + + df: pd.DataFrame = pd.DataFrame(data_rows) + df = df[self.input_features + [self.label_name]] + return df + + def _verify(self) -> None: + """Verify the integrity of the dataset. + + Raises: + RuntimeError: if ``download=False`` but dataset is missing or checksum fails + """ + # Check if the extracted files already exist + pathname = os.path.join(self.root, self.collection_id) + if os.path.exists(pathname): + return + + # Check if the zip files have already been downloaded + pathname = os.path.join(self.root, self.collection_id) + ".tar.gz" + if os.path.exists(pathname): + self._extract() + return + + # Check if the user requested to download the dataset + if not self.download: + raise RuntimeError( + f"Dataset not found in `root={self.root}` and `download=False`, " + "either specify a different `root` directory or use `download=True` " + "to automatically download the dataset." + ) + + # Download the dataset + self._download() + self._extract() + + def _extract(self) -> None: + """Extract the dataset.""" + pathname = os.path.join(self.root, self.collection_id) + ".tar.gz" + extract_archive(pathname, self.root) + + def _download(self, api_key: Optional[str] = None) -> None: + """Download the dataset and extract it. + + Args: + api_key: a RadiantEarth MLHub API key to use for downloading the dataset + + Raises: + RuntimeError: if download doesn't work correctly or checksums don't match + """ + download_radiant_mlhub_collection(self.collection_id, self.root, api_key) + filename = os.path.join(self.root, self.collection_id) + ".tar.gz" + extract_archive(filename, self.root)