diff --git a/.codespellignore b/.codespellignore index e69de29b..35f2ebe7 100644 --- a/.codespellignore +++ b/.codespellignore @@ -0,0 +1 @@ +ned diff --git a/.readthedocs.yml b/.readthedocs.yml index 176e874e..6debd292 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -31,6 +31,8 @@ python: path: stactools_landsat/ - method: pip path: stactools_planet/ + - method: pip + path: stactools_threedep/ - method: pip path: stactools_browse/ - method: pip diff --git a/README.md b/README.md index f72058c7..20ecca86 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ See the [documentation page](https://stactools.readthedocs.io/en/latest/) for th | `stactools_planet` | Methods and commands for working with planet data | | `stactools_landsat` | Methods and commands for working with landsat data | | `stactools_sentinel2` | Methods and commands for working with Sentinel 2 data | +| `stactools_threedep` | Methods and commands for working with 3DEP (formerly NED) elevation data | | `stactools_browse` | Contains a command for launching stac-browser against a local STAC | Subpackages are symlinked to the `stactools` directory in this repo to allow them to be importable for python running at the top level directory of the repository clone. diff --git a/docker/Dockerfile b/docker/Dockerfile index 8cfa9e16..7554adde 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -82,6 +82,10 @@ RUN pip install -r /tmp/planet/requirements.txt COPY stactools_sentinel2/requirements.txt /tmp/sentinel2/requirements.txt RUN pip install -r /tmp/sentinel2/requirements.txt +# 3DEP +COPY stactools_threedep/requirements.txt /tmp/threedep/requirements.txt +RUN pip install -r /tmp/threedep/requirements.txt + # Jupyter RUN pip install jupyter==1.0.0 diff --git a/scripts/env b/scripts/env index d2469d08..e76a346d 100755 --- a/scripts/env +++ b/scripts/env @@ -9,6 +9,7 @@ STACTOOLS_SUBPACKAGE_DIRS=( "stactools_landsat" "stactools_planet" "stactools_naip" + "stactools_threedep" "stactools_browse" "stactools_sentinel2" ); @@ -22,6 +23,7 @@ STACTOOLS_COVERAGE_DIRS=( "stactools_landsat/stactools/landsat" "stactools_planet/stactools/planet" "stactools_naip/stactools/naip" + "stactools_threedep/stactools/threedep" "stactools_browse/stactools/browse" "stactools_sentinel2/stactools/sentinel2" ); \ No newline at end of file diff --git a/setup.py b/setup.py index 40456c62..9c9c4687 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ def requirement_name(self): Subpackage('landsat', is_extra=True), Subpackage('naip', is_extra=True), Subpackage('planet', is_extra=True), + Subpackage('threedep', is_extra=True), Subpackage('browse', is_extra=True), Subpackage('sentinel2', is_extra=True) ] diff --git a/stactools/threedep b/stactools/threedep new file mode 120000 index 00000000..a760db57 --- /dev/null +++ b/stactools/threedep @@ -0,0 +1 @@ +../stactools_threedep/stactools/threedep/ \ No newline at end of file diff --git a/stactools_threedep/README.md b/stactools_threedep/README.md new file mode 100644 index 00000000..6f4da98e --- /dev/null +++ b/stactools_threedep/README.md @@ -0,0 +1,46 @@ +# stactools.threedep + +The 3D Elevation Program (3DEP), formerly known as the National Elevation Dataset (NED), is elevation data for the United States. +Because `3dep` isn't a valid Python package name, this package is named `stactools.threedep`. + +## Usage + +3DEP items are identified by two attributes: `product` and `id`. +`product` is a string that corresponds to the nominal resolution of the data. +Options are: + +- "1": 1 arc-second DEMs +- "13" 1/3 arc-second DEMs + +`id` is a lat-lon identifier, e.g. `n41w106`. + +### Command line + +To build a `pystac.Collection` directly from AWS into a directory named `usgs-3dep-stac`: + +```bash +stac threedep create-collection usgs-3dep-stac +``` + +If you want to two-step the process, you can download the metadata first: + +```bash +stac threedep download-metadata usgs-3dep-metadata +stac threedep create-collection usgs-3dep-stac --source usgs-3dep-metadata +``` + +### API + +To create an item from AWS, use `stactools.threedep.stac.create_item`: + +```python +from stactools.threedep import stac +item = stac.create_item_from_product_and_id("1", "n41w106") +``` + +You can also create an item directly from the href of a metadata XML file anyhwere: + +```python +from stactools.threedep import stac +item = stac.create_item("ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n41w106/USGS_1_n41w106.xml") +``` diff --git a/stactools_threedep/requirements.txt b/stactools_threedep/requirements.txt new file mode 100644 index 00000000..1b958f15 --- /dev/null +++ b/stactools_threedep/requirements.txt @@ -0,0 +1 @@ +boto3==1.17.40 \ No newline at end of file diff --git a/stactools_threedep/setup.py b/stactools_threedep/setup.py new file mode 100644 index 00000000..15972910 --- /dev/null +++ b/stactools_threedep/setup.py @@ -0,0 +1,47 @@ +import os +from os.path import (basename, splitext) +from imp import load_source +from setuptools import setup, find_namespace_packages +from glob import glob +import io + +name = 'stactools_threedep' +description = ("Subpackage for working with 3DEP data in stactools, " + "a command line tool and Python library for working with STAC.") + +__version__ = load_source( + 'stactools.threedep.version', + os.path.join(os.path.dirname(__file__), + 'stactools/threedep/version.py')).__version__ + +here = os.path.abspath(os.path.dirname(__file__)) + +# get the dependencies and installs +with io.open(os.path.join(here, 'requirements.txt'), encoding='utf-8') as f: + install_requires = [line.split(' ')[0] for line in f.read().split('\n')] + +# Add stactools subpackage dependencies +install_requires.extend(['stactools_core=={}'.format(__version__)]) + +with open(os.path.join(here, 'README.md')) as readme_file: + readme = readme_file.read() + +setup(name=name, + description=description, + version=__version__, + long_description=readme, + long_description_content_type="text/markdown", + author="Pete Gadomski", + author_email='pgadomski@element84.com', + url='https://github.com/stac-utils/stactools.git', + packages=find_namespace_packages(), + py_modules=[ + splitext(basename(path))[0] for path in glob('stactools/*.py') + ], + include_package_data=False, + install_requires=install_requires, + license="Apache Software License 2.0", + keywords=[ + 'stactools', 'psytac', '3DEP', 'NED', 'DEM', 'elevation', 'raster', + 'catalog', 'STAC' + ]) diff --git a/stactools_threedep/stactools/threedep/__init__.py b/stactools_threedep/stactools/threedep/__init__.py new file mode 100644 index 00000000..d9031152 --- /dev/null +++ b/stactools_threedep/stactools/threedep/__init__.py @@ -0,0 +1,13 @@ +import stactools.core + +from stactools.threedep.metadata import Metadata + +stactools.core.use_fsspec() + + +def register_plugin(registry): + from stactools.threedep import commands + registry.register_subcommand(commands.create_threedep_command) + + +__all__ = [Metadata] diff --git a/stactools_threedep/stactools/threedep/commands.py b/stactools_threedep/stactools/threedep/commands.py new file mode 100644 index 00000000..3351cc29 --- /dev/null +++ b/stactools_threedep/stactools/threedep/commands.py @@ -0,0 +1,128 @@ +import click +import os.path + +from pystac import Catalog, Collection, Extent, CatalogType, STAC_IO + +from stactools.threedep.constants import (PRODUCTS, DESCRIPTION, USGS_PROVIDER, + USGS_FTP_BASE, USGS_3DEP_ID) +from stactools.threedep import utils, stac + + +def create_threedep_command(cli): + """Creates the threedep command line utility.""" + @cli.group("threedep", short_help="Work with USGS 3DEP elevation data.") + def threedep(): + pass + + @threedep.command( + "create-catalog", + short_help="Create a STAC catalog for existing USGS 3DEP data") + @click.argument("destination") + @click.option("-s", + "--source", + help="The href of a directory tree containing metadata", + default=None) + @click.option("-i", + "--id", + multiple=True, + help="Ids to fetch. If not provided, will fetch all IDs.") + @click.option("--quiet/--no-quiet", default=False) + def create_catalog_command(destination, source, id, quiet): + """Creates a relative published 3DEP catalog in DESTINATION. + + If SOURCE is not provided, will use the metadata in AWS. SOURCE is + expected to be a directory tree mirroring the structure on USGS, so + it is best created using `stac threedep download-metadata`. + """ + base_ids = id # not sure how to rename arguments in click + collections = {} + items = {} + for product in PRODUCTS: + items[product] = [] + if base_ids: + ids = base_ids + else: + ids = utils.fetch_ids(product) + for id in ids: + item = stac.create_item_from_product_and_id( + product, id, source) + items[product].append(item) + if not quiet: + print(item.id) + extent = Extent.from_items(items[product]) + if product == "1": + title = "1 arc-second" + description = "USGS 3DEP 1 arc-second DEMs" + elif product == "13": + title = "1/3 arc-second" + description = "USGS 3DEP 1/3 arc-second DEMs" + else: + raise NotImplementedError + collection = Collection( + id=f"{USGS_3DEP_ID}-{product}", + title=title, + keywords=["USGS", "3DEP", "NED", "DEM", "elevation"], + providers=[USGS_PROVIDER], + description=description, + extent=extent, + license="PDDL-1.0") + collections[product] = collection + catalog = Catalog(id=USGS_3DEP_ID, + description=DESCRIPTION, + title="USGS 3DEP DEMs", + catalog_type=CatalogType.RELATIVE_PUBLISHED) + for product, collection in collections.items(): + catalog.add_child(collection) + collection.add_items(items[product]) + catalog.generate_subcatalogs("${threedep:region}") + catalog.normalize_hrefs(destination) + catalog.save() + catalog.validate() + + @threedep.command("download-metadata", + short_help="Download all metadata for USGS 3DEP data") + @click.argument("destination") + @click.option("-i", + "--id", + multiple=True, + help="Ids to fetch. If not provided, will fetch all IDs.") + @click.option("--quiet/--no-quiet", default=False) + def download_metadata_command(destination, id, quiet): + """Creates a 3DEP collection in DESTINATION.""" + base_ids = id # not sure how to rename arguments in click + for product in PRODUCTS: + if base_ids: + ids = base_ids + else: + ids = utils.fetch_ids(product) + for id in ids: + path = utils.path(product, + id, + extension="xml", + base=destination) + if os.path.exists(path): + if not quiet: + print("{} exists, skipping download...".format(path)) + continue + os.makedirs(os.path.dirname(path), exist_ok=True) + source_path = utils.path(product, + id, + extension="xml", + base=USGS_FTP_BASE) + if not quiet: + print("{} -> {}".format(source_path, path)) + text = STAC_IO.read_text(source_path) + with open(path, "w") as f: + f.write(text) + + @threedep.command( + "fetch-ids", + short_help="Fetch all product ids and print them to stdout") + @click.argument("product") + @click.option("--usgs-ftp/--no-usgs-ftp", + default=False, + help="Fetch from the USGS FTP instead of AWS.") + def fetch_ids_command(product: str, usgs_ftp: bool): + """Fetches product ids and prints them to stdout.""" + for id in utils.fetch_ids(product, use_usgs_ftp=usgs_ftp): + print(id) diff --git a/stactools_threedep/stactools/threedep/constants.py b/stactools_threedep/stactools/threedep/constants.py new file mode 100644 index 00000000..15a53c70 --- /dev/null +++ b/stactools_threedep/stactools/threedep/constants.py @@ -0,0 +1,28 @@ +# flake8: noqa + +from pyproj import CRS +from pystac import Provider + +USGS_3DEP_ID = "usgs-3dep" +THREEDEP_EPSG = 5498 +THREEDEP_CRS = CRS.from_epsg(THREEDEP_EPSG) +PRODUCTS = ["1", "13"] +LICENSE = "PDDL-1.0" + +DESCRIPTION = """The USGS 3D Elevation Program (3DEP) Datasets from The National Map are the primary elevation data product produced and distributed by the USGS. The 3DEP program provides a variety of resolution raster elevation data of the conterminous United States, Alaska, Hawaii, and the island territories. Some of the data sets such as the 1/3rd arc-second and 1 arc-second data set are derived from diverse source data sets that are processed to a specification with a consistent resolution, coordinate system, elevation units, and horizontal and vertical datums. These seamless DEMs were referred to as the National Elevation Dataset (NED) from about 2000 through 2015 at which time they became the seamless DEM layers under the 3DEP program and the NED name and system were retired. Other 3DEP products include one-meter DEMs produced exclusively from high resolution light detection and ranging (lidar) source data and five-meter DEMs in Alaska as well as various source datasets including the lidar point cloud and interferometric synthetic aperture radar (Ifsar) digital surface models and intensity images. All 3DEP products are public domain. The 3DEP program is the logical result of the maturation of the long-standing USGS elevation program, which for many years concentrated on production of topographic map quadrangle-based digital elevation models. The 3DEP data serves as the elevation layer of The National Map, and provides basic elevation information for earth science studies and mapping applications in the United States. + +The seamless DEM layers under the 3DEP program are a multi-resolution dataset that is updated continuously to integrate newly available, improved elevation source data. Seamless DEM data layers under the 3DEP program are available nationally at grid spacings of 1 arc-second (approximately 30 meters) for the conterminous United States, and at 1/3, 1/9 arc-seconds (approximately 10 and 3 meters, respectively) and 1 meter for parts of the United States. Most seamless DEM data for Alaska is available at 2-arc-second (about 60 meters) grid spacing, where only lower resolution source data exist. Part of Alaska is available at the 1/3 arc-second, 1 arc-second and 5 meter resolution. Efforts are continuing to have full coverage of the 5 meter elevation data over Alaska in the next couple of years. +""" + +USGS_PROVIDER = Provider( + name="USGS", + roles=["producer", "processor", "host"], + url="https://www.usgs.gov/core-science-systems/ngp/3dep") + +USGS_FTP_SERVER = "rockyftp.cr.usgs.gov" +USGS_FTP_BASE = f"ftp://{USGS_FTP_SERVER}/vdelivery/Datasets/Staged/Elevation" +AWS_BUCKET = "prd-tnm" +AWS_PREFIX = "StagedProducts/Elevation" +AWS_BASE = f"https://{AWS_BUCKET}.s3.amazonaws.com/{AWS_PREFIX}" + +DEFAULT_BASE = AWS_BASE \ No newline at end of file diff --git a/stactools_threedep/stactools/threedep/metadata.py b/stactools_threedep/stactools/threedep/metadata.py new file mode 100644 index 00000000..b5d1f82c --- /dev/null +++ b/stactools_threedep/stactools/threedep/metadata.py @@ -0,0 +1,238 @@ +from __future__ import annotations +import datetime +from typing import Union, Optional +from xml.etree import ElementTree +from xml.etree.ElementTree import Element + +from shapely.geometry import box, mapping +from pystac import Asset, MediaType, Link + +from stactools.core import io +from stactools.core.io import ReadHrefModifier +from stactools.core.projection import reproject_geom +from stactools.threedep.constants import THREEDEP_CRS, THREEDEP_EPSG, DEFAULT_BASE +from stactools.threedep import utils + + +class Metadata: + """3DEP file metadata.""" + @classmethod + def from_href( + cls, + href: str, + read_href_modifier: Optional[ReadHrefModifier] = None) -> Metadata: + """Creates a metadata from an href to the XML metadata file.""" + text = io.read_text(href, read_href_modifier) + element_tree = ElementTree.fromstring(text) + return cls(element_tree) + + @classmethod + def from_product_and_id(cls, + product: str, + id: str, + base: str = None) -> Metadata: + """Creates a Metadata from a product and id.""" + if base is None: + base = DEFAULT_BASE + href = utils.path(product, id, extension="xml", base=base) + return cls.from_href(href) + + def __init__(self, xml: Element): + """Creates a new metadata object from XML metadata.""" + self.title = xml.findtext("./idinfo/citation/citeinfo/title") + self.description = xml.findtext("./idinfo/descript/abstract") + bounding = xml.find("./idinfo/spdom/bounding") + self.minx = bounding.findtext("./westbc") + self.miny = bounding.findtext("./southbc") + self.maxx = bounding.findtext("./eastbc") + self.maxy = bounding.findtext("./northbc") + self.pubdate = xml.findtext("./idinfo/citation/citeinfo/pubdate") + self.begdate = xml.findtext( + "./idinfo/timeperd/timeinfo/rngdates/begdate") + self.enddate = xml.findtext( + "./idinfo/timeperd/timeinfo/rngdates/enddate") + self.current = xml.findtext("./idinfo/timeperd/current") + self.rowcount = xml.findtext("./spdoinfo/rastinfo/rowcount") + self.colcount = xml.findtext("./spdoinfo/rastinfo/colcount") + self.latres = xml.findtext("./spref/horizsys/geograph/latres") + self.longres = xml.findtext("./spref/horizsys/geograph/longres") + tiff_href = xml.findtext( + "./distinfo/stdorder/digform/digtopt/onlinopt/computer/networka/networkr" + ) + parts = tiff_href.split('/')[-4:] + self.product = parts[0] + self.id = parts[2] + + @property + def stac_id(self) -> str: + """Returns the STAC ID of this metadata. + + This is the id plus the product, e.g. if the filename of the tif is + "USGS_1_n40w105.tif", then the STAC id is "n40w105-1". + """ + return "{}-{}".format(self.id, self.product) + + @property + def geometry(self) -> dict: + """Returns this item's geometry in WGS84.""" + original_bbox = [ + float(self.minx), + float(self.miny), + float(self.maxx), + float(self.maxy) + ] + return reproject_geom(THREEDEP_CRS, "EPSG:4326", + mapping(box(*original_bbox))) + + @property + def datetime(self) -> Union[datetime.datetime, None]: + """Returns the collection publication datetime.""" + if self.current == "publication date": + return _format_date(self.pubdate) + else: + raise NotImplementedError + + @property + def start_datetime(self) -> Union[datetime.datetime, None]: + """Returns the start datetime for this record. + + This can be a while ago, since the national elevation dataset was + originally derived from direct survey data. + """ + return _format_date(self.begdate) + + @property + def end_datetime(self) -> Union[datetime.datetime, None]: + """Returns the end datetime for this record.""" + return _format_date(self.enddate, end_of_year=True) + + @property + def gsd(self) -> float: + """Returns the nominal ground sample distance from these metadata.""" + if self.product == "1": + return 30 + elif self.product == "13": + return 10 + else: + raise NotImplementedError + + def data_asset(self, base: str = DEFAULT_BASE) -> Asset: + """Returns the data asset (aka the tiff file).""" + return Asset(href=self._asset_href_with_extension(base, "tif"), + title=self.title, + description=self.description, + media_type=MediaType.COG, + roles=["data"]) + + def metadata_asset(self, base: str = DEFAULT_BASE) -> Asset: + """Returns the data asset (aka the tiff file).""" + return Asset(href=self._asset_href_with_extension(base, "xml"), + media_type=MediaType.XML, + roles=["metadata"]) + + def thumbnail_asset(self, base: str = DEFAULT_BASE) -> Asset: + """Returns the thumbnail asset.""" + return Asset(href=self._asset_href_with_extension(base, "jpg"), + media_type=MediaType.JPEG, + roles=["thumbnail"]) + + def gpkg_asset(self, base: str = DEFAULT_BASE) -> Asset: + """Returns the geopackage asset.""" + return Asset(href=self._asset_href_with_extension(base, + "gpkg", + id_only=True), + media_type=MediaType.GEOPACKAGE, + roles=["metadata"]) + + def via_link(self, base: str = DEFAULT_BASE) -> Link: + """Returns the via link for this file.""" + return Link("via", self._asset_href_with_extension(base, "xml")) + + @property + def projection_extension_dict(self) -> dict: + """Returns a dictionary of values to be applied to the projection extension.""" + shape = [int(self.rowcount), int(self.colcount)] + transform = [ + float(self.longres), + 0.0, + float(self.minx), + 0.0, + -float(self.latres), + float(self.maxy), + 0.0, + 0.0, + 1.0, + ] + return { + "epsg": THREEDEP_EPSG, + "shape": shape, + "transform": transform, + } + + @property + def region(self) -> str: + """Returns this objects 3dep "region". + + Region is defined as a 10x10 lat/lon box that nominally contains this item. + E.g. for n41w106, the region would be n40w110. This is used mostly for + creating subcatalogs for STACBrowser. + """ + import math + n_or_s = self.id[0] + lat = float(self.id[1:3]) + if n_or_s == "s": + lat = -lat + lat = math.floor(lat / 10) * 10 + e_or_w = self.id[3] + lon = float(self.id[4:]) + if e_or_w == "w": + lon = -lon + lon = math.floor(lon / 10) * 10 + return f"{n_or_s}{abs(lat)}{e_or_w}{abs(lon)}" + + def _asset_href_with_extension(self, + base: str, + extension: str, + id_only: bool = False) -> str: + if base is None: + base = DEFAULT_BASE + return utils.path(self.product, + self.id, + base=base, + extension=extension, + id_only=id_only) + + +def _format_date(date: str, + end_of_year: bool = False) -> Union[datetime.datetime, None]: + if len(date) == 4: + year = int(date) + if end_of_year: + month = 12 + day = 31 + else: + month = 1 + day = 1 + if year < 1800 or year > datetime.date.today().year: + return None # There's some bad metadata in the USGS records + else: + return datetime.datetime(year, + month, + day, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc) + elif len(date) == 8: + year = int(date[0:4]) + month = int(date[4:6]) + day = int(date[6:8]) + return datetime.datetime(year, + month, + day, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc) + else: + return None diff --git a/stactools_threedep/stactools/threedep/stac.py b/stactools_threedep/stactools/threedep/stac.py new file mode 100644 index 00000000..5432297f --- /dev/null +++ b/stactools_threedep/stactools/threedep/stac.py @@ -0,0 +1,50 @@ +from typing import Optional + +from pystac import Item +from shapely.geometry import shape + +from stactools.core.io import ReadHrefModifier +from stactools.threedep.metadata import Metadata +from stactools.threedep.constants import DEFAULT_BASE + + +def create_item(href: str, + read_href_modifier: Optional[ReadHrefModifier] = None, + base: str = DEFAULT_BASE) -> Item: + """Creates a STAC item from an href to an XML metadata file.""" + metadata = Metadata.from_href(href, read_href_modifier) + return create_item_from_metadata(metadata, base) + + +def create_item_from_product_and_id(product: str, + id: str, + base: str = DEFAULT_BASE) -> Item: + """Creates a STAC item from a product (e.g. "1") and an ID (e.g. "n41w106").""" + metadata = Metadata.from_product_and_id(product, id) + return create_item_from_metadata(metadata, base) + + +def create_item_from_metadata(metadata: Metadata, base: DEFAULT_BASE) -> Item: + """Creates a STAC item from Metadata.""" + geometry = metadata.geometry + bbox = list(shape(geometry).bounds) + item = Item(id=metadata.stac_id, + geometry=geometry, + bbox=bbox, + datetime=metadata.datetime, + properties={}) + start_datetime = metadata.start_datetime + end_datetime = metadata.end_datetime + if start_datetime and end_datetime: + item.common_metadata.start_datetime = start_datetime + item.common_metadata.end_datetime = end_datetime + item.common_metadata.gsd = metadata.gsd + item.links.append(metadata.via_link(base)) + item.assets["data"] = metadata.data_asset(base) + item.assets["metadata"] = metadata.metadata_asset(base) + item.assets["thumbnail"] = metadata.thumbnail_asset(base) + item.assets["gpkg"] = metadata.gpkg_asset(base) + item.ext.enable("projection") + item.ext.projection.apply(**metadata.projection_extension_dict) + item.properties["threedep:region"] = metadata.region + return item diff --git a/stactools_threedep/stactools/threedep/utils.py b/stactools_threedep/stactools/threedep/utils.py new file mode 100644 index 00000000..1604189f --- /dev/null +++ b/stactools_threedep/stactools/threedep/utils.py @@ -0,0 +1,68 @@ +import os.path + +import boto3 +from ftplib import FTP + +from stactools.threedep.constants import USGS_FTP_SERVER, AWS_BUCKET, AWS_PREFIX +from stactools.threedep import utils + + +def fetch_ids(product: str, use_usgs_ftp: bool = False) -> [str]: + """Returns all ids for the given product.""" + if use_usgs_ftp: + return _fetch_ids_from_usgs_ftp(product) + else: + return _fetch_ids_from_aws(product) + + +def _fetch_ids_from_usgs_ftp(product: str) -> [str]: + directory = f"vdelivery/Datasets/Staged/Elevation/{product}/TIFF" + ftp = FTP(USGS_FTP_SERVER) + ftp.login() + ids = [ + file_name for (file_name, _) in ftp.mlsd(directory) + if not file_name.startswith(".") + ] + ftp.close() + return ids + + +def _fetch_ids_from_aws(product: str) -> [str]: + path = os.path.dirname(utils.path(product, "")) + prefix = os.path.join(AWS_PREFIX, path) + client = boto3.client("s3") + paginator = client.get_paginator("list_objects_v2") + page_iterator = paginator.paginate(Bucket=AWS_BUCKET, Prefix=prefix) + filtered_iterator = page_iterator.search( + "Contents[?ends_with(Key, `.xml`)].Key") + # The main tif/xml files for each id are named simply like "USGS_1_n41w106.tif". + # If there's been updates, older versions will have a datetime on the end, e.g. + # "USGS_1_n41w106_20210330.tif" or something similar. By splitting on underscores, + # we're hoping to catch only the "main" files. + return [ + os.path.basename(os.path.dirname(key)) for key in filtered_iterator + if len(os.path.basename(key).split("_")) == 3 + ] + + +def path(product: str, + id: str, + base: str = None, + extension: str = None, + id_only: bool = False) -> str: + """Returns the subpath for this product and id. + + E.g. path("1", "n41w106") == "1/TIFF/n41w106/USGS_1_n41w106" + E.g. path("1", "n41w106", extension="tif") == "1/TIFF/n41w106/USGS_1_n41w106.tif" + E.g. path("1", "n41w106", base="/base/dir") == "/base/dir/1/TIFF/n41w106/USGS_1_n41w106" + E.g. path("1", "n41w106", id_only=True) == "/base/dir/1/TIFF/n41w106/n41w106" + """ + if id_only: + path = "{}/TIFF/{}/{}".format(product, id, id) + else: + path = "{}/TIFF/{}/USGS_{}_{}".format(product, id, product, id) + if extension: + path = "{}.{}".format(path, extension) + if base: + path = os.path.join(base, path) + return path diff --git a/stactools_threedep/stactools/threedep/version.py b/stactools_threedep/stactools/threedep/version.py new file mode 100644 index 00000000..6fa6b34c --- /dev/null +++ b/stactools_threedep/stactools/threedep/version.py @@ -0,0 +1,2 @@ +__version__ = '0.1.4' +"""Library version""" diff --git a/tests/data-files/threedep/base/1/TIFF/n40w106/USGS_1_n40w106.xml b/tests/data-files/threedep/base/1/TIFF/n40w106/USGS_1_n40w106.xml new file mode 100644 index 00000000..d0b0aad3 --- /dev/null +++ b/tests/data-files/threedep/base/1/TIFF/n40w106/USGS_1_n40w106.xml @@ -0,0 +1,207 @@ + + + + + + + U.S. Geological Survey + 20180328 + USGS 1 arc-second n40w106 1 x 1 degree + raster digital data + + Reston, VA + U.S. Geological Survey + + https://nationalmap.gov/elevation.html + https://nationalmap.gov/viewer.html + + + + This tile of the 3D Elevation Program (3DEP) seamless products is 1 arc-second resolution. 3DEP data serve as the elevation layer of The National Map, and provide basic elevation information for Earth science studies and mapping applications in the United States. Scientists and resource managers use 3DEP data for global change research, hydrologic modeling, resource monitoring, mapping and visualization, and many other applications. 3DEP data compose an elevation dataset that consists of seamless layers and a high resolution layer. Each of these layers consists of the best available raster elevation data of the conterminous United States, Alaska, Hawaii, territorial islands, Mexico and Canada. 3DEP data are updated continually as new data become available. Seamless 3DEP data are derived from diverse source data that are processed to a common coordinate system and unit of vertical measure. These data are distributed in geographic coordinates in units of decimal degrees, and in conformance with the North American Datum of 1983 (NAD 83). All elevation values are in meters and, over the conterminous United States, are referenced to the North American Vertical Datum of 1988 (NAVD 88). The vertical reference will vary in other areas. Seamless 3DEP data are available nationally (except for Alaska) at resolutions of 1 arc-second (approximately 30 meters) and 1/3 arc-second (approximately 10 meters). In most of Alaska, only lower resolution source data are available. As a result, most seamless 3DEP data for Alaska are at 2 arc-second (approximately 60 meters) grid spacing. Part of Alaska is available at the 1 and 1/3 arc-second resolutions from interferometric synthetic aperture radar (ifsar) collections starting in 2010. Plans are in place for collection of statewide ifsar in Alaska. All 3DEP products are public domain. + The 3DEP data serves as the elevation layer of The National Map, and provides basic elevation information for Earth science studies and mapping applications in the United States. The data are utilized by the scientific and resource management communities for global change research, hydrologic modeling, resource monitoring, mapping and visualization applications. + This tile of seamless 3DEP dataset is 1 arc-second resolution, TIFF file format, and covers a 1 degree block. The geographic area of coverage is described below in the Bounding Coordinates. Additional information for the 3DEP products may be found on https://nationalmap.gov/elevation.html. Data may be downloaded through The National Map Viewer: https://nationalmap.gov/viewer.html. Direct links for direct access by automated services are provided in the "Distribution" section of this metadata. + + + + + 1947 + 2016 + + + publication date + + + In work + As needed + + + + -106.001666667082 + -104.998333333716 + 40.001666666984 + 38.9983333336173 + + + + ISO 19115 Topic Category + elevation + 1-degree DEM + 1 arc-second DEM + + + NGDA Portfolio Themes + Elevation + + + 3D Elevation Program (3DEP) + 3D Elevation Program + 3DEP + National Elevation Dataset + NED + Elevation + Grid + Light Detection And Ranging + LIDAR + Interferometric Synthetic Aperture Radar + IFSAR + High Resolution + Topographic Surface + Topography + Bare Earth + Hydro-Flattened + Terrain Elevation + Cartography + DEM + Digital Elevation Model + Geographic Information System + Digital Mapping + Digital Terrain Model + Geodata + GIS + Mapping + Raster + USGS + U.S. Geological Survey + + + Geographic Names Information System + US + United States + + None. Any downloading and use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Insure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. + There is no guarantee or warranty concerning the accuracy of the data. Users should be aware that temporal changes may have occurred since these data were collected and that some parts of these data may no longer represent actual surface conditions. Users should not use these data for critical applications without a full awareness of their limitations. Acknowledgement of the originating agencies would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. Please refer to https://www.usgs.gov/privacy.html for the USGS disclaimer. + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n40w106/USGS_1_n40w106.jpg + Thumbnail JPG image + JPEG + + Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + + Raster + + Pixel + 3612 + 3612 + 1 + + + + + + 0.00027777778 + 0.00027777778 + Decimal degrees + + + North American Datum of 1983 + Geodetic Reference System 80 + 6378137.000000 + 298.2572221 + + + + + North American Vertical Datum of 1988 + 1.000000 + meters + Implicit coordinate + + + + + + The attribute summary can be found in the Data Dictionary (http://nationalmap.gov/3DEP/3dep_prodmetadata.html). + A link to the 3DEP Data Dictionary describing information that is included in the spatial metadata shapefiles acquired with the DEM is provided at the end of the Metadata Information section of this document. + + + + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + Downloadable Data + Although these data have been processed successfully on a computer system at the U.S. Geological Survey (USGS), no warranty expressed or implied is made regarding the display or utility of the data for other purposes, nor on all computer systems, nor shall the act of distribution constitute any such warranty. The USGS or the U.S. Government shall not be held liable for improper or incorrect use of the data described and/or contained herein. + + + + TIFF + 0.048295 + + + + + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n40w106/USGS_1_n40w106.tif + + + + + + NONE + + + + 20180328 + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + http://nationalmap.gov/3DEP/3dep_prodmetadata.html + 3D Elevation Program (3DEP) Metadata + + + \ No newline at end of file diff --git a/tests/data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml b/tests/data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml new file mode 100644 index 00000000..ac40b2b7 --- /dev/null +++ b/tests/data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml @@ -0,0 +1,201 @@ + + + + + + + U.S. Geological Survey + 20150717 + USGS 1 arc-second n41w106 1 x 1 degree + raster digital data + + Reston, VA + U.S. Geological Survey + + http://ned.usgs.gov/ + http://nationalmap.gov/viewer.html + + + + This tile of the 3D Elevation Program (3DEP) seamless products is 1 arc-second resolution. 3DEP data serve as the elevation layer of The National Map, and provide basic elevation information for Earth science studies and mapping applications in the United States. Scientists and resource managers use 3DEP data for global change research, hydrologic modeling, resource monitoring, mapping and visualization, and many other applications. 3DEP data compose an elevation dataset that consists of seamless layers and a high resolution layer. Each of these layers consists of the best available raster elevation data of the conterminous United States, Alaska, Hawaii, territorial islands, Mexico and Canada. 3DEP data are updated continually as new data become available. Seamless 3DEP data are derived from diverse source data that are processed to a common coordinate system and unit of vertical measure. These data are distributed in geographic coordinates in units of decimal degrees, and in conformance with the North American Datum of 1983 (NAD 83). All elevation values are in meters and, over the conterminous United States, are referenced to the North American Vertical Datum of 1988 (NAVD 88). The vertical reference will vary in other areas. Seamless 3DEP data are available nationally (except for Alaska) at resolutions of 1 arc-second (approximately 30 meters) and 1/3 arc-second (approximately 10 meters). In most of Alaska, only lower resolution source data are available. As a result, most seamless 3DEP data for Alaska are at 2 arc-second (approximately 60 meters) grid spacing. Part of Alaska is available at the 1 and 1/3 arc-second resolutions from interferometric synthetic aperture radar (ifsar) collections starting in 2010. Plans are in place for collection of statewide ifsar in Alaska. All 3DEP products are public domain. + The 3DEP data serves as the elevation layer of The National Map, and provides basic elevation information for Earth science studies and mapping applications in the United States. The data are utilized by the scientific and resource management communities for global change research, hydrologic modeling, resource monitoring, mapping and visualization applications. + This tile of seamless 3DEP dataset is 1 arc-second resolution, TIFF file format, and covers a 1 degree block. The geographic area of coverage is described below in the Bounding Coordinates. Additional information for the 3DEP products may be found on https://nationalmap.gov/elevation.html. Data may be downloaded through The National Map Viewer: https://nationalmap.gov/viewer.html. Direct links for direct access by automated services are provided in the "Distribution" section of this metadata. + + + + + 1948 + 2013 + + + publication date + + + In work + As needed + + + + -106.001666667082 + -104.998333333716 + 41.0016666667842 + 39.9983333334175 + + + + ISO 19115 Topic Category + elevation + 1-degree DEM + 1 arc-second DEM + + + NGDA Portfolio Themes + Elevation + + + 3D Elevation Program (3DEP) + 3D Elevation Program + 3DEP + National Elevation Dataset + NED + Elevation + Grid + Light Detection And Ranging + LIDAR + Interferometric Synthetic Aperture Radar + IFSAR + High Resolution + Topographic Surface + Topography + Bare Earth + Hydro-Flattened + Terrain Elevation + Cartography + DEM + Digital Elevation Model + Geographic Information System + Digital Mapping + Digital Terrain Model + Geodata + GIS + Mapping + Raster + USGS + U.S. Geological Survey + + + Geographic Names Information System + US + United States + + None. Any downloading and use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Insure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. + There is no guarantee or warranty concerning the accuracy of the data. Users should be aware that temporal changes may have occurred since these data were collected and that some parts of these data may no longer represent actual surface conditions. Users should not use these data for critical applications without a full awareness of its limitations. Acknowledgement of the originating agencies would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n41w106/USGS_1_n41w106.jpg + Thumbnail JPG image + JPEG + + Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + + Raster + + Pixel + 3612 + 3612 + 1 + + + + + + 0.00027777778 + 0.00027777778 + Decimal degrees + + + North American Datum of 1983 + Geodetic Reference System 80 + 6378137.000000 + 298.2572221 + + + + + North American Vertical Datum of 1988 + 1.000000 + meters + Implicit coordinate + + + + + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + Downloadable Data + Although these data have been processed successfully on a computer system at the U.S. Geological Survey, no warranty, expressed or implied, is made by either regarding the utility of the data on any system, nor shall the act of distribution constitute any such warranty. The USGS will warranty the delivery of this product in computer-readable format and will offer appropriate adjustment of credit when the product is determined unreadable by correctly adjusted computer peripherals, or when the physical medium is delivered in damaged condition. Requests for adjustments of credit must be made within 90 days from the date of this shipment from the ordering site. + + + + TIFF + 0.047916 + + + + + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n41w106/USGS_1_n41w106.tif + + + + + + NONE + + + + 20150717 + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + http://ned.usgs.gov/metadata.html + National Elevation Dataset (NED) Metadata + + + \ No newline at end of file diff --git a/tests/data-files/threedep/base/13/TIFF/n40w106/USGS_13_n40w106.xml b/tests/data-files/threedep/base/13/TIFF/n40w106/USGS_13_n40w106.xml new file mode 100644 index 00000000..d84c769d --- /dev/null +++ b/tests/data-files/threedep/base/13/TIFF/n40w106/USGS_13_n40w106.xml @@ -0,0 +1,207 @@ + + + + + + + U.S. Geological Survey + 20180328 + USGS 1/3 arc-second n40w106 1 x 1 degree + raster digital data + + Reston, VA + U.S. Geological Survey + + https://nationalmap.gov/elevation.html + https://nationalmap.gov/viewer.html + + + + This tile of the 3D Elevation Program (3DEP) seamless products is 1/3 arc-second resolution. 3DEP data serve as the elevation layer of The National Map, and provide basic elevation information for Earth science studies and mapping applications in the United States. Scientists and resource managers use 3DEP data for global change research, hydrologic modeling, resource monitoring, mapping and visualization, and many other applications. 3DEP data compose an elevation dataset that consists of seamless layers and a high resolution layer. Each of these layers consists of the best available raster elevation data of the conterminous United States, Alaska, Hawaii, territorial islands, Mexico and Canada. 3DEP data are updated continually as new data become available. Seamless 3DEP data are derived from diverse source data that are processed to a common coordinate system and unit of vertical measure. These data are distributed in geographic coordinates in units of decimal degrees, and in conformance with the North American Datum of 1983 (NAD 83). All elevation values are in meters and, over the conterminous United States, are referenced to the North American Vertical Datum of 1988 (NAVD 88). The vertical reference will vary in other areas. Seamless 3DEP data are available nationally (except for Alaska) at resolutions of 1 arc-second (approximately 30 meters) and 1/3 arc-second (approximately 10 meters). In most of Alaska, only lower resolution source data are available. As a result, most seamless 3DEP data for Alaska are at 2 arc-second (approximately 60 meters) grid spacing. Part of Alaska is available at the 1 and 1/3 arc-second resolutions from interferometric synthetic aperture radar (ifsar) collections starting in 2010. Plans are in place for collection of statewide ifsar in Alaska. All 3DEP products are public domain. + The 3DEP data serves as the elevation layer of The National Map, and provides basic elevation information for Earth science studies and mapping applications in the United States. The data are utilized by the scientific and resource management communities for global change research, hydrologic modeling, resource monitoring, mapping and visualization applications. + This tile of seamless 3DEP dataset is 1/3 arc-second resolution, TIFF file format, and covers a 1 degree block. The geographic area of coverage is described below in the Bounding Coordinates. Additional information for the 3DEP products may be found on https://nationalmap.gov/elevation.html. Data may be downloaded through The National Map Viewer: https://nationalmap.gov/viewer.html. Direct links for direct access by automated services are provided in the "Distribution" section of this metadata. + + + + + 1947 + 2016 + + + publication date + + + In work + As needed + + + + -106.000555555594 + -104.999444445204 + 40.0005555554954 + 38.999444445106 + + + + ISO 19115 Topic Category + elevation + 1-degree DEM + 13 arc-second DEM + + + NGDA Portfolio Themes + Elevation + + + 3D Elevation Program (3DEP) + 3D Elevation Program + 3DEP + National Elevation Dataset + NED + Elevation + Grid + Light Detection And Ranging + LIDAR + Interferometric Synthetic Aperture Radar + IFSAR + High Resolution + Topographic Surface + Topography + Bare Earth + Hydro-Flattened + Terrain Elevation + Cartography + DEM + Digital Elevation Model + Geographic Information System + Digital Mapping + Digital Terrain Model + Geodata + GIS + Mapping + Raster + USGS + U.S. Geological Survey + + + Geographic Names Information System + US + United States + + None. Any downloading and use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Insure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. + There is no guarantee or warranty concerning the accuracy of the data. Users should be aware that temporal changes may have occurred since these data were collected and that some parts of these data may no longer represent actual surface conditions. Users should not use these data for critical applications without a full awareness of their limitations. Acknowledgement of the originating agencies would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. Please refer to https://www.usgs.gov/privacy.html for the USGS disclaimer. + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/TIFF/n40w106/USGS_13_n40w106.jpg + Thumbnail JPG image + JPEG + + Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + + Raster + + Pixel + 10812 + 10812 + 1 + + + + + + 0.000092592593 + 0.000092592593 + Decimal degrees + + + North American Datum of 1983 + Geodetic Reference System 80 + 6378137.000000 + 298.2572221 + + + + + North American Vertical Datum of 1988 + 1.000000 + meters + Implicit coordinate + + + + + + The attribute summary can be found in the Data Dictionary (http://nationalmap.gov/3DEP/3dep_prodmetadata.html). + A link to the 3DEP Data Dictionary describing information that is included in the spatial metadata shapefiles acquired with the DEM is provided at the end of the Metadata Information section of this document. + + + + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + Downloadable Data + Although these data have been processed successfully on a computer system at the U.S. Geological Survey (USGS), no warranty expressed or implied is made regarding the display or utility of the data for other purposes, nor on all computer systems, nor shall the act of distribution constitute any such warranty. The USGS or the U.S. Government shall not be held liable for improper or incorrect use of the data described and/or contained herein. + + + + TIFF + 0.382822 + + + + + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/TIFF/n40w106/USGS_13_n40w106.tif + + + + + + NONE + + + + 20180328 + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + http://nationalmap.gov/3DEP/3dep_prodmetadata.html + 3D Elevation Program (3DEP) Metadata + + + \ No newline at end of file diff --git a/tests/data-files/threedep/base/13/TIFF/n41w106/USGS_13_n41w106.xml b/tests/data-files/threedep/base/13/TIFF/n41w106/USGS_13_n41w106.xml new file mode 100644 index 00000000..2a6cc5f8 --- /dev/null +++ b/tests/data-files/threedep/base/13/TIFF/n41w106/USGS_13_n41w106.xml @@ -0,0 +1,201 @@ + + + + + + + U.S. Geological Survey + 20150717 + USGS 1/3 arc-second n41w106 1 x 1 degree + raster digital data + + Reston, VA + U.S. Geological Survey + + http://ned.usgs.gov/ + http://nationalmap.gov/viewer.html + + + + This tile of the 3D Elevation Program (3DEP) seamless products is 1/3 arc-second resolution. 3DEP data serve as the elevation layer of The National Map, and provide basic elevation information for Earth science studies and mapping applications in the United States. Scientists and resource managers use 3DEP data for global change research, hydrologic modeling, resource monitoring, mapping and visualization, and many other applications. 3DEP data compose an elevation dataset that consists of seamless layers and a high resolution layer. Each of these layers consists of the best available raster elevation data of the conterminous United States, Alaska, Hawaii, territorial islands, Mexico and Canada. 3DEP data are updated continually as new data become available. Seamless 3DEP data are derived from diverse source data that are processed to a common coordinate system and unit of vertical measure. These data are distributed in geographic coordinates in units of decimal degrees, and in conformance with the North American Datum of 1983 (NAD 83). All elevation values are in meters and, over the conterminous United States, are referenced to the North American Vertical Datum of 1988 (NAVD 88). The vertical reference will vary in other areas. Seamless 3DEP data are available nationally (except for Alaska) at resolutions of 1 arc-second (approximately 30 meters) and 1/3 arc-second (approximately 10 meters). In most of Alaska, only lower resolution source data are available. As a result, most seamless 3DEP data for Alaska are at 2 arc-second (approximately 60 meters) grid spacing. Part of Alaska is available at the 1 and 1/3 arc-second resolutions from interferometric synthetic aperture radar (ifsar) collections starting in 2010. Plans are in place for collection of statewide ifsar in Alaska. All 3DEP products are public domain. + The 3DEP data serves as the elevation layer of The National Map, and provides basic elevation information for Earth science studies and mapping applications in the United States. The data are utilized by the scientific and resource management communities for global change research, hydrologic modeling, resource monitoring, mapping and visualization applications. + This tile of seamless 3DEP dataset is 1/3 arc-second resolution, TIFF file format, and covers a 1 degree block. The geographic area of coverage is described below in the Bounding Coordinates. Additional information for the 3DEP products may be found on https://nationalmap.gov/elevation.html. Data may be downloaded through The National Map Viewer: https://nationalmap.gov/viewer.html. Direct links for direct access by automated services are provided in the "Distribution" section of this metadata. + + + + + 1948 + 2013 + + + publication date + + + In work + As needed + + + + -106.000555555594 + -104.999444445204 + 41.0005555552955 + 39.9994444449061 + + + + ISO 19115 Topic Category + elevation + 1-degree DEM + 13 arc-second DEM + + + NGDA Portfolio Themes + Elevation + + + 3D Elevation Program (3DEP) + 3D Elevation Program + 3DEP + National Elevation Dataset + NED + Elevation + Grid + Light Detection And Ranging + LIDAR + Interferometric Synthetic Aperture Radar + IFSAR + High Resolution + Topographic Surface + Topography + Bare Earth + Hydro-Flattened + Terrain Elevation + Cartography + DEM + Digital Elevation Model + Geographic Information System + Digital Mapping + Digital Terrain Model + Geodata + GIS + Mapping + Raster + USGS + U.S. Geological Survey + + + Geographic Names Information System + US + United States + + None. Any downloading and use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Insure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. + There is no guarantee or warranty concerning the accuracy of the data. Users should be aware that temporal changes may have occurred since these data were collected and that some parts of these data may no longer represent actual surface conditions. Users should not use these data for critical applications without a full awareness of its limitations. Acknowledgement of the originating agencies would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/TIFF/n41w106/USGS_13_n41w106.jpg + Thumbnail JPG image + JPEG + + Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + + Raster + + Pixel + 10812 + 10812 + 1 + + + + + + 0.000092592593 + 0.000092592593 + Decimal degrees + + + North American Datum of 1983 + Geodetic Reference System 80 + 6378137.000000 + 298.2572221 + + + + + North American Vertical Datum of 1988 + 1.000000 + meters + Implicit coordinate + + + + + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + Downloadable Data + Although these data have been processed successfully on a computer system at the U.S. Geological Survey, no warranty, expressed or implied, is made by either regarding the utility of the data on any system, nor shall the act of distribution constitute any such warranty. The USGS will warranty the delivery of this product in computer-readable format and will offer appropriate adjustment of credit when the product is determined unreadable by correctly adjusted computer peripherals, or when the physical medium is delivered in damaged condition. Requests for adjustments of credit must be made within 90 days from the date of this shipment from the ordering site. + + + + TIFF + 0.377854 + + + + + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/TIFF/n41w106/USGS_13_n41w106.tif + + + + + + NONE + + + + 20150717 + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + http://ned.usgs.gov/metadata.html + National Elevation Dataset (NED) Metadata + + + \ No newline at end of file diff --git a/tests/data-files/threedep/one-offs/USGS_1_n19w090.xml b/tests/data-files/threedep/one-offs/USGS_1_n19w090.xml new file mode 100644 index 00000000..31dcab47 --- /dev/null +++ b/tests/data-files/threedep/one-offs/USGS_1_n19w090.xml @@ -0,0 +1,197 @@ + + + + + + + U.S. Geological Survey + 2013 + USGS 1 arc-second n19w090 1 x 1 degree + raster digital data + + Reston, VA + U.S. Geological Survey + + http://ned.usgs.gov/ + http://nationalmap.gov/viewer.html + + + + This tile of the 3D Elevation Program (3DEP) seamless products is 1 arc-second resolution. 3DEP data serve as the elevation layer of The National Map, and provide basic elevation information for Earth science studies and mapping applications in the United States. Scientists and resource managers use 3DEP data for global change research, hydrologic modeling, resource monitoring, mapping and visualization, and many other applications. 3DEP data compose an elevation dataset that consists of seamless layers and a high resolution layer. Each of these layers consists of the best available raster elevation data of the conterminous United States, Alaska, Hawaii, territorial islands, Mexico and Canada. 3DEP data are updated continually as new data become available. Seamless 3DEP data are derived from diverse source data that are processed to a common coordinate system and unit of vertical measure. These data are distributed in geographic coordinates in units of decimal degrees, and in conformance with the North American Datum of 1983 (NAD 83). All elevation values are in meters and, over the conterminous United States, are referenced to the North American Vertical Datum of 1988 (NAVD 88). The vertical reference will vary in other areas. Seamless 3DEP data are available nationally (except for Alaska) at resolutions of 1 arc-second (approximately 30 meters) and 1/3 arc-second (approximately 10 meters). In most of Alaska, only lower resolution source data are available. As a result, most seamless 3DEP data for Alaska are at 2 arc-second (approximately 60 meters) grid spacing. Part of Alaska is available at the 1 and 1/3 arc-second resolutions from interferometric synthetic aperture radar (ifsar) collections starting in 2010. Plans are in place for collection of statewide ifsar in Alaska. All 3DEP products are public domain. + The 3DEP data serves as the elevation layer of The National Map, and provides basic elevation information for Earth science studies and mapping applications in the United States. The data are utilized by the scientific and resource management communities for global change research, hydrologic modeling, resource monitoring, mapping and visualization applications. + This tile of seamless 3DEP dataset is 1 arc-second resolution, TIFF file format, and covers a 1 degree block. The geographic area of coverage is described below in the Bounding Coordinates. Additional information for the 3DEP products may be found on https://nationalmap.gov/elevation.html. Data may be downloaded through The National Map Viewer: https://nationalmap.gov/viewer.html. Direct links for direct access by automated services are provided in the "Distribution" section of this metadata. + + + + + 19990201 + 20131101 + + + publication date + + + In work + As Needed + + + + -90.00166666667 + -88.99833333334 + 19.00166666666 + 17.99833333333 + + + + ISO 19115 Topic Category + elevation + 1-degree DEM + 1 arc-second DEM + + + NGDA Portfolio Themes + Elevation + + + 3D Elevation Program (3DEP) + 3D Elevation Program + 3DEP + National Elevation Dataset + NED + Elevation + Grid + Light Detection And Ranging + LIDAR + Interferometric Synthetic Aperture Radar + IFSAR + High Resolution + Topographic Surface + Topography + Bare Earth + Hydro-Flattened + Terrain Elevation + Cartography + DEM + Digital Elevation Model + Geographic Information System + Digital Mapping + Digital Terrain Model + Geodata + GIS + Mapping + Raster + USGS + U.S. Geological Survey + + + Geographic Names Information System + US + United States + + None. Any downloading and use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Insure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. + There is no guarantee or warranty concerning the accuracy of the data. Users should be aware that temporal changes may have occurred since these data were collected and that some parts of these data may no longer represent actual surface conditions. Users should not use these data for critical applications without a full awareness of its limitations. Acknowledgement of the originating agencies would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n19w090/USGS_1_n19w090.jpg + Thumbnail JPG image + JPEG + + Acknowledgement of the originating agencies would be appreciated in products derived from these data. + + + Raster + + Pixel + 3612 + 3612 + 1 + + + + + + 0.00001 + 0.00001 + Decimal degrees + + + North American Datum of 1983 + Geodetic Reference System 80 + 6378137.000000 + 298.2572221 + + + + + North American Vertical Datum of 1988 + 0.001 + meters + Implicit coordinate + + + + + + + + U.S. Geological Survey + + + mailing and physical +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + Downloadable Data + Although these data have been processed successfully on a computer system at the U.S. Geological Survey, no warranty, expressed or implied, is made by either regarding the utility of the data on any system, nor shall the act of distribution constitute any such warranty. The USGS will warranty the delivery of this product in computer-readable format and will offer appropriate adjustment of credit when the product is determined unreadable by correctly adjusted computer peripherals, or when the physical medium is delivered in damaged condition. Requests for adjustments of credit must be made within 90 days from the date of this shipment from the ordering site. + + + + TIFF + 0.011651 + + + + + + ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/TIFF/n19w090/USGS_1_n19w090.tif + + + + + + NONE + + + + 2013 + + + + U.S. Geological Survey + + + mailing and physical address +
USGS National Geospatial Program Office
+
12201 Sunrise Valley Drive
+ Reston + VA + 20192 + USA + + 1-888-ASK-USGS (1-888-275-8747) + Monday through Friday 8:00 AM to 4:00 PM Eastern Time Zone USA + Please visit http://www.usgs.gov/ask/ to contact us. + + + FGDC Content Standard for Digital Geospatial Metadata + FGDC-STD-001-1998 + + \ No newline at end of file diff --git a/tests/threedep/__init__.py b/tests/threedep/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/threedep/test_commands.py b/tests/threedep/test_commands.py new file mode 100644 index 00000000..fde726e9 --- /dev/null +++ b/tests/threedep/test_commands.py @@ -0,0 +1,28 @@ +import os.path +from tempfile import TemporaryDirectory + +import pystac + +from stactools.threedep.commands import create_threedep_command +from tests.utils import CliTestCase, TestData + + +class CreateCollectionTest(CliTestCase): + def create_subcommand_functions(self): + return [create_threedep_command] + + def test_create_collection(self): + path = TestData.get_path("data-files/threedep/base") + with TemporaryDirectory() as directory: + result = self.run_command([ + "threedep", "create-catalog", directory, "--id", "n41w106", + "--id", "n40w106", "--quiet", "--source", path + ]) + self.assertEqual(result.exit_code, + 0, + msg="\n{}".format(result.output)) + catalog = pystac.read_file(os.path.join(directory, "catalog.json")) + item_ids = set([item.id for item in catalog.get_all_items()]) + self.assertEqual( + item_ids, + set(["n40w106-1", "n40w106-13", "n41w106-1", "n41w106-13"])) diff --git a/tests/threedep/test_stac.py b/tests/threedep/test_stac.py new file mode 100644 index 00000000..51bb7715 --- /dev/null +++ b/tests/threedep/test_stac.py @@ -0,0 +1,174 @@ +import datetime +import unittest + +from tests.utils import TestData +from stactools.threedep import stac +from stactools.threedep.constants import USGS_FTP_BASE + + +class CreateItemTest(unittest.TestCase): + def test_create_item_1(self): + path = TestData.get_path( + "data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml") + item = stac.create_item(path) + self.assertEqual(item.id, "n41w106-1") + self.assertTrue(item.geometry is not None) + self.assertTrue(item.bbox is not None) + self.assertEqual( + item.datetime, + datetime.datetime(2015, + 7, + 17, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc)) + self.assertEqual( + item.common_metadata.start_datetime, + datetime.datetime(1948, + 1, + 1, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc)) + self.assertEqual( + item.common_metadata.end_datetime, + datetime.datetime(2013, + 12, + 31, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc)) + self.assertEqual(item.common_metadata.gsd, 30) + + data = item.assets["data"] + self.assertEqual(data.href, + ("https://prd-tnm.s3.amazonaws.com/StagedProducts" + "/Elevation/1/TIFF/n41w106/USGS_1_n41w106.tif")) + self.assertEqual(data.title, "USGS 1 arc-second n41w106 1 x 1 degree") + self.assertTrue( + data.description.startswith( + "This tile of the 3D Elevation Program (3DEP)")) + self.assertTrue( + data.media_type, + "image/tiff; application=geotiff; profile=cloud-optimized") + self.assertTrue(data.roles, ["data"]) + + data = item.assets["metadata"] + self.assertEqual(data.href, + ("https://prd-tnm.s3.amazonaws.com/StagedProducts" + "/Elevation/1/TIFF/n41w106/USGS_1_n41w106.xml")) + self.assertTrue(data.title is None) + self.assertTrue(data.description is None) + self.assertEqual(data.media_type, "application/xml") + self.assertEqual(data.roles, ["metadata"]) + + data = item.assets["thumbnail"] + self.assertEqual( + data.href, + ("https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/" + "1/TIFF/n41w106/USGS_1_n41w106.jpg")) + self.assertTrue(data.title is None) + self.assertTrue(data.description is None) + self.assertEqual(data.media_type, "image/jpeg") + self.assertEqual(data.roles, ["thumbnail"]) + + data = item.assets["gpkg"] + self.assertEqual( + data.href, + ("https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/" + "1/TIFF/n41w106/n41w106.gpkg")) + self.assertTrue(data.title is None) + self.assertTrue(data.description is None) + self.assertEqual(data.media_type, "application/geopackage+sqlite3") + self.assertEqual(data.roles, ["metadata"]) + + link = next(link for link in item.links if link.rel == "via") + self.assertTrue(link is not None) + self.assertEqual(link.target, + ("https://prd-tnm.s3.amazonaws.com/StagedProducts" + "/Elevation/1/TIFF/n41w106/USGS_1_n41w106.xml")) + + item.ext.enable("projection") + self.assertEqual(item.ext.projection.epsg, 5498) + self.assertEqual(item.ext.projection.shape, [3612, 3612]) + self.assertEqual(item.ext.projection.transform, [ + 0.00027777778, 0.0, -106.001666667082, 0.0, -0.00027777778, + 41.0016666667842, 0.0, 0.0, 1.0 + ]) + + self.assertEqual(item.properties["threedep:region"], "n40w110") + + item.validate() + + def test_create_item_1_weird_date(self): + path = TestData.get_path( + "data-files/threedep/one-offs/USGS_1_n19w090.xml") + item = stac.create_item(path) + self.assertEqual( + item.datetime, + datetime.datetime(2013, + 1, + 1, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc)) + + def test_create_item_13(self): + path = TestData.get_path( + "data-files/threedep/base/13/TIFF/n41w106/USGS_13_n41w106.xml") + item = stac.create_item(path) + self.assertEqual(item.id, "n41w106-13") + self.assertEqual(item.common_metadata.gsd, 10) + + def test_create_item_with_base(self): + path = TestData.get_path( + "data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml") + item = stac.create_item(path, base=USGS_FTP_BASE) + data = item.assets["data"] + self.assertEqual( + data.href, ("ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged" + "/Elevation/1/TIFF/n41w106/USGS_1_n41w106.tif")) + data = item.assets["metadata"] + self.assertEqual( + data.href, ("ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged" + "/Elevation/1/TIFF/n41w106/USGS_1_n41w106.xml")) + data = item.assets["thumbnail"] + self.assertEqual( + data.href, + ("ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/" + "1/TIFF/n41w106/USGS_1_n41w106.jpg")) + link = next(link for link in item.links if link.rel == "via") + self.assertTrue(link is not None) + self.assertEqual( + link.target, + ("ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged" + "/Elevation/1/TIFF/n41w106/USGS_1_n41w106.xml")) + + def test_create_item_from_product_and_id(self): + path = TestData.get_path("data-files/threedep/base") + item = stac.create_item_from_product_and_id("1", "n41w106", path) + item.validate() + + def test_read_href_modifier(self): + did_it = False + + def modify_href(href: str) -> str: + nonlocal did_it + did_it = True + return href + + path = TestData.get_path( + "data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml") + _ = stac.create_item(path, modify_href) + self.assertTrue(did_it) + + def test_explicit_none_goes_to_aws(self): + path = TestData.get_path( + "data-files/threedep/base/1/TIFF/n41w106/USGS_1_n41w106.xml") + item0 = stac.create_item(path) + item1 = stac.create_item(path, base=None) + self.assertEqual(item0.to_dict(), item1.to_dict()) diff --git a/tests/threedep/test_utils.py b/tests/threedep/test_utils.py new file mode 100644 index 00000000..5e10075c --- /dev/null +++ b/tests/threedep/test_utils.py @@ -0,0 +1,17 @@ +import unittest + +from stactools.threedep import utils + + +class PathTest(unittest.TestCase): + def test_simple(self): + path = utils.path("1", "n41w106") + self.assertEqual(path, "1/TIFF/n41w106/USGS_1_n41w106") + + def test_extension(self): + path = utils.path("1", "n41w106", extension="foo") + self.assertEqual(path, "1/TIFF/n41w106/USGS_1_n41w106.foo") + + def test_base(self): + path = utils.path("1", "n41w106", base="foo") + self.assertEqual(path, "foo/1/TIFF/n41w106/USGS_1_n41w106")