From ef5425ca522ae077da7c77196086ae54e65eee40 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Tue, 11 Feb 2025 09:51:13 -0800 Subject: [PATCH 1/9] add make-env-files to pre-commit --- .pre-commit-config.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 33b72118..15605eb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,14 @@ repos: args: [--autofix] - id: trailing-whitespace + - repo: local + hooks: + - id: make-env-files + name: Make env files + entry: python environments/make-env-files.py + language: system + require_serial: true + - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: @@ -35,14 +43,14 @@ repos: - id: numpydoc-validation - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.3 + rev: v0.9.6 hooks: - id: ruff args: [--fix] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.1 + rev: v1.15.0 hooks: - id: mypy additional_dependencies: From 02285d242b40d7b9ba1fbc42ccf5892adf74c01b Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Tue, 11 Feb 2025 09:51:19 -0800 Subject: [PATCH 2/9] update docs --- docs/source/getting-started.rst | 2 +- docs/source/index.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 0f0ecbaf..320eca22 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -12,7 +12,7 @@ Get Started in 4 Steps 4. Consult the :doc:`user-reference` for complete details on using the package. -Finally, if you're not already familiar with `NetworkX`_ and `GeoPandas`_, make sure you read their user guides as OSMnx uses their data structures and functionality. +Finally, if you're not already familiar with `NetworkX`_ and `GeoPandas`_, make sure you read their user guides as OSMnx uses their data structures. .. _introducing-osmnx: diff --git a/docs/source/index.rst b/docs/source/index.rst index 81c0c399..77ac302f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -49,8 +49,8 @@ OSMnx is open source and licensed under the MIT license. OpenStreetMap's open da .. _license: https://www.openstreetmap.org/copyright -Documentation -------------- +User Guides +----------- .. toctree:: :maxdepth: 1 From 407f1fb08df0a6f71b5bbd8fa02f63abc925fc45 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 12:18:13 -0800 Subject: [PATCH 3/9] dev version --- osmnx/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnx/_version.py b/osmnx/_version.py index b7356a3e..0994fc5b 100644 --- a/osmnx/_version.py +++ b/osmnx/_version.py @@ -1,3 +1,3 @@ """OSMnx package version information.""" -__version__ = "2.0.1" +__version__ = "2.0.2dev" From e87c55741d42fbb62795a69f778e10c03c7674d6 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 12:18:55 -0800 Subject: [PATCH 4/9] do not save response to cache if it contains a server remark --- osmnx/_http.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/osmnx/_http.py b/osmnx/_http.py index 2ab749ba..438f0013 100644 --- a/osmnx/_http.py +++ b/osmnx/_http.py @@ -55,6 +55,9 @@ def _save_to_cache( if not ok: # pragma: no cover msg = "Did not save to cache because HTTP status code is not OK" utils.log(msg, level=lg.WARNING) + elif isinstance(response_json, dict) and ("remark" in response_json): # pragma: no cover + msg = f"Did not save to cache because response contains remark: {response_json['remark']!r}" + utils.log(msg, lg.WARNING) else: # create the folder on the disk if it doesn't already exist cache_folder = Path(settings.cache_folder) @@ -118,19 +121,8 @@ def _retrieve_from_cache(url: str) -> dict[str, Any] | list[dict[str, Any]] | No # return cached response for this url if exists, otherwise return None cache_filepath = _url_in_cache(url) if cache_filepath is not None: - response_json: dict[str, Any] | list[dict[str, Any]] = json.loads( - cache_filepath.read_text(encoding="utf-8"), - ) - - # return None if there is a server remark in the cached response - if isinstance(response_json, dict) and ("remark" in response_json): # pragma: no cover - msg = ( - f"Ignoring cache file {str(cache_filepath)!r} because " - f"it contains a remark: {response_json['remark']!r}" - ) - utils.log(msg, lg.WARNING) - return None - + response_json: dict[str, Any] | list[dict[str, Any]] + response_json = json.loads(cache_filepath.read_text(encoding="utf-8")) msg = f"Retrieved response from cache file {str(cache_filepath)!r}" utils.log(msg, lg.INFO) return response_json From b9e5481e6ec121176141ed1f9db27065011e3866 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 12:19:23 -0800 Subject: [PATCH 5/9] update default/error request pause durations --- osmnx/_overpass.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osmnx/_overpass.py b/osmnx/_overpass.py index 9d5a9c6c..7839283f 100644 --- a/osmnx/_overpass.py +++ b/osmnx/_overpass.py @@ -141,8 +141,8 @@ def _get_network_filter(network_type: str) -> str: def _get_overpass_pause( base_endpoint: str, *, - recursive_delay: float = 5, - default_duration: float = 60, + recursive_delay: float = 4, + default_duration: float = 50, ) -> float: """ Retrieve a pause duration from the Overpass API status endpoint. @@ -432,7 +432,7 @@ def _overpass_request( data: OrderedDict[str, Any], *, pause: float | None = None, - error_pause: float = 60, + error_pause: float = 55, ) -> dict[str, Any]: """ Send a HTTP POST request to the Overpass API and return response. From 0cfe2f4037de6003db816cc0107fe6275fb9fc30 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 17:45:09 -0800 Subject: [PATCH 6/9] fix bug in parsing utc time for pausing --- osmnx/_overpass.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/osmnx/_overpass.py b/osmnx/_overpass.py index 7839283f..280b238a 100644 --- a/osmnx/_overpass.py +++ b/osmnx/_overpass.py @@ -183,7 +183,7 @@ def _get_overpass_pause( response_text = response.text except RequestsConnectionError as e: # pragma: no cover # cannot reach status endpoint: log error and return default duration - msg = f"Unable to query {url}, {e}" + msg = f"Unable to reach {url}, {e}" utils.log(msg, level=lg.ERROR) return default_duration @@ -209,7 +209,7 @@ def _get_overpass_pause( if status_first_part == "Slot": utc_time_str = status.split(" ")[3] pattern = "%Y-%m-%dT%H:%M:%SZ," - utc_time = dt.datetime.strptime(utc_time_str, pattern).astimezone(dt.timezone.utc) + utc_time = dt.datetime.strptime(utc_time_str, pattern).replace(tzinfo=dt.timezone.utc) utc_now = dt.datetime.now(tz=dt.timezone.utc) seconds = int(np.ceil((utc_time - utc_now).total_seconds())) pause = max(seconds, 1) @@ -465,11 +465,11 @@ def _overpass_request( # pause then request this URL if pause is None: - this_pause = _get_overpass_pause(settings.overpass_url) + pause = _get_overpass_pause(settings.overpass_url) hostname = _http._hostname_from_url(url) - msg = f"Pausing {this_pause} second(s) before making HTTP POST request to {hostname!r}" + msg = f"Pausing {pause} second(s) before making HTTP POST request to {hostname!r}" utils.log(msg, level=lg.INFO) - time.sleep(this_pause) + time.sleep(pause) # transmit the HTTP POST request msg = f"Post {prepared_url} with timeout={settings.requests_timeout}" @@ -484,14 +484,13 @@ def _overpass_request( # handle 429 and 504 errors by pausing then recursively re-trying request if response.status_code in {429, 504}: # pragma: no cover - this_pause = error_pause + _get_overpass_pause(settings.overpass_url) msg = ( f"{hostname!r} responded {response.status_code} {response.reason}: " - f"we'll retry in {this_pause} secs" + f"we'll retry in {error_pause} secs" ) utils.log(msg, level=lg.WARNING) - time.sleep(this_pause) - return _overpass_request(data, pause=pause, error_pause=error_pause) + time.sleep(error_pause) + return _overpass_request(data) response_json = _http._parse_response(response) if not isinstance(response_json, dict): # pragma: no cover From edc6ad9cf2cd12a00e18f3f72e9952ba8f5017fa Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 19:10:05 -0800 Subject: [PATCH 7/9] use tomli for python<3.11 --- environments/make-env-files.py | 16 ++++++++++++---- environments/requirements/requirements-all.txt | 1 + environments/requirements/requirements-tests.txt | 1 + environments/tests/env-ci.yml | 1 + environments/tests/env-test-minimum-deps.yml | 1 + .../tests/requirements-test-latest-deps.txt | 1 + 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/environments/make-env-files.py b/environments/make-env-files.py index 65f84f7b..598822dc 100644 --- a/environments/make-env-files.py +++ b/environments/make-env-files.py @@ -9,7 +9,13 @@ from typing import Any from packaging.requirements import Requirement -from tomllib import load as tomllib_load + +try: + # for python >=3.11 + from tomllib import load as toml_load +except ImportError: + # for python <3.11 + from tomli import load as toml_load # path to package's pyproject and the config json file PYPROJECT_PATH = "./pyproject.toml" @@ -60,7 +66,7 @@ def make_requirement( return str(requirement) -def make_file(env: dict[str, Any]) -> None: +def make_file(env: dict[str, Any], pyproject: dict[str, Any]) -> None: """ Write a conda environment yaml file or pip requirements.txt file. @@ -68,6 +74,8 @@ def make_file(env: dict[str, Any]) -> None: ---------- env An environment configuration dictionary. + pyproject + A parsed pyproject.toml file's contents. """ depends_on = [] output_path = Path(env["output_path"]) @@ -117,10 +125,10 @@ def make_file(env: dict[str, Any]) -> None: if __name__ == "__main__": # load the pyproject.toml and the environments.json config files with Path(PYPROJECT_PATH).open("rb") as f: - pyproject = tomllib_load(f) + pyproject = toml_load(f) with Path(ENVS_CONFIG_PATH).open("rb") as f: envs = json_load(f) # make each environment/requirements file as configured for env in envs: - make_file(env) + make_file(env, pyproject) diff --git a/environments/requirements/requirements-all.txt b/environments/requirements/requirements-all.txt index b520418d..f13e95f8 100644 --- a/environments/requirements/requirements-all.txt +++ b/environments/requirements/requirements-all.txt @@ -32,6 +32,7 @@ shapely>=2.0 sphinx-autodoc-typehints sphinx>=7 statsmodels +tomli twine typeguard validate-pyproject diff --git a/environments/requirements/requirements-tests.txt b/environments/requirements/requirements-tests.txt index 209ec5e4..b76fbda1 100644 --- a/environments/requirements/requirements-tests.txt +++ b/environments/requirements/requirements-tests.txt @@ -4,3 +4,4 @@ pre-commit pytest pytest-cov typeguard +tomli diff --git a/environments/tests/env-ci.yml b/environments/tests/env-ci.yml index 77b0cc3a..414b7ac3 100644 --- a/environments/tests/env-ci.yml +++ b/environments/tests/env-ci.yml @@ -26,6 +26,7 @@ dependencies: - shapely>=2.0 - sphinx-autodoc-typehints - sphinx>=7 + - tomli - twine - typeguard - validate-pyproject diff --git a/environments/tests/env-test-minimum-deps.yml b/environments/tests/env-test-minimum-deps.yml index 769d6f60..464af6dc 100644 --- a/environments/tests/env-test-minimum-deps.yml +++ b/environments/tests/env-test-minimum-deps.yml @@ -22,4 +22,5 @@ dependencies: - scikit-learn==0.23.* - scipy==1.5.* - shapely==2.0.* + - tomli - typeguard diff --git a/environments/tests/requirements-test-latest-deps.txt b/environments/tests/requirements-test-latest-deps.txt index fc8dc7dd..e0ef8bf7 100644 --- a/environments/tests/requirements-test-latest-deps.txt +++ b/environments/tests/requirements-test-latest-deps.txt @@ -17,4 +17,5 @@ rio-vrt>=0.3 scikit-learn>=0.23 scipy>=1.5 shapely>=2.0 +tomli typeguard From eae3d46eae9e02a234e8484fb89c8dc1f4f4e8ba Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 20:14:11 -0800 Subject: [PATCH 8/9] remove unnecessary args --- osmnx/_nominatim.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/osmnx/_nominatim.py b/osmnx/_nominatim.py index 771dba52..73ce065f 100644 --- a/osmnx/_nominatim.py +++ b/osmnx/_nominatim.py @@ -145,12 +145,7 @@ def _nominatim_request( ) utils.log(msg, level=lg.WARNING) time.sleep(error_pause) - return _nominatim_request( - params, - request_type=request_type, - pause=pause, - error_pause=error_pause, - ) + return _nominatim_request(params, request_type=request_type) response_json = _http._parse_response(response) if not isinstance(response_json, list): From 91f892cc474081154f5ab1feb07ac1422393b7d8 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 14 Feb 2025 20:14:16 -0800 Subject: [PATCH 9/9] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27c04332..1085c4d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,10 @@ ## 2.0.2 (TBD) -- improve docstrings (#1272 #1274) +- fix bug in parsing time when calculating pause duration between requests (#1277) - fix bug where consolidate_intersections function would mutate the passed-in graph (#1273) - provide user-friendly error message if consolidate_intersections is run more than once (#1273) +- improve docstrings (#1272 #1274) ## 2.0.1 (2025-01-01)