Merge pull request #1277 from gboeing/fix

Fix bug in calculating pause duration between requests

.pre-commit-config.yaml

@@ -23,6 +23,14 @@ repos:
         args: [--autofix]
       - id: trailing-whitespace
 
+  - repo: local
+    hooks:
+      - id: make-env-files
+        name: Make env files
+        entry: python environments/make-env-files.py
+        language: system
+        require_serial: true
+
   - repo: https://github.com/adrienverge/yamllint
     rev: v1.35.1
     hooks:
@@ -35,14 +43,14 @@ repos:
       - id: numpydoc-validation
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.3
+    rev: v0.9.6
     hooks:
       - id: ruff
         args: [--fix]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.1
+    rev: v1.15.0
     hooks:
       - id: mypy
         additional_dependencies:

CHANGELOG.md

@@ -2,9 +2,10 @@
 
 ## 2.0.2 (TBD)
 
-- improve docstrings (#1272 #1274)
+- fix bug in parsing time when calculating pause duration between requests (#1277)
 - fix bug where consolidate_intersections function would mutate the passed-in graph (#1273)
 - provide user-friendly error message if consolidate_intersections is run more than once (#1273)
+- improve docstrings (#1272 #1274)
 
 ## 2.0.1 (2025-01-01)
 

docs/source/getting-started.rst

@@ -12,7 +12,7 @@ Get Started in 4 Steps
 
 4. Consult the :doc:`user-reference` for complete details on using the package.
 
-Finally, if you're not already familiar with `NetworkX`_ and `GeoPandas`_, make sure you read their user guides as OSMnx uses their data structures and functionality.
+Finally, if you're not already familiar with `NetworkX`_ and `GeoPandas`_, make sure you read their user guides as OSMnx uses their data structures.
 
 .. _introducing-osmnx:
 

docs/source/index.rst

@@ -49,8 +49,8 @@ OSMnx is open source and licensed under the MIT license. OpenStreetMap's open da
 .. _license: https://www.openstreetmap.org/copyright
 
 
-Documentation
--------------
+User Guides
+-----------
 
 .. toctree::
    :maxdepth: 1

environments/make-env-files.py

@@ -9,7 +9,13 @@
 from typing import Any
 
 from packaging.requirements import Requirement
-from tomllib import load as tomllib_load
+
+try:
+    # for python >=3.11
+    from tomllib import load as toml_load
+except ImportError:
+    # for python <3.11
+    from tomli import load as toml_load
 
 # path to package's pyproject and the config json file
 PYPROJECT_PATH = "./pyproject.toml"
@@ -60,14 +66,16 @@ def make_requirement(
     return str(requirement)
 
 
-def make_file(env: dict[str, Any]) -> None:
+def make_file(env: dict[str, Any], pyproject: dict[str, Any]) -> None:
     """
     Write a conda environment yaml file or pip requirements.txt file.
 
     Parameters
     ----------
     env
         An environment configuration dictionary.
+    pyproject
+        A parsed pyproject.toml file's contents.
     """
     depends_on = []
     output_path = Path(env["output_path"])
@@ -117,10 +125,10 @@ def make_file(env: dict[str, Any]) -> None:
 if __name__ == "__main__":
     # load the pyproject.toml and the environments.json config files
     with Path(PYPROJECT_PATH).open("rb") as f:
-        pyproject = tomllib_load(f)
+        pyproject = toml_load(f)
     with Path(ENVS_CONFIG_PATH).open("rb") as f:
         envs = json_load(f)
 
     # make each environment/requirements file as configured
     for env in envs:
-        make_file(env)
+        make_file(env, pyproject)

environments/requirements/requirements-all.txt

@@ -32,6 +32,7 @@ shapely>=2.0
 sphinx-autodoc-typehints
 sphinx>=7
 statsmodels
+tomli
 twine
 typeguard
 validate-pyproject

environments/requirements/requirements-tests.txt

@@ -4,3 +4,4 @@ pre-commit
 pytest
 pytest-cov
 typeguard
+tomli

environments/tests/env-ci.yml

@@ -26,6 +26,7 @@ dependencies:
   - shapely>=2.0
   - sphinx-autodoc-typehints
   - sphinx>=7
+  - tomli
   - twine
   - typeguard
   - validate-pyproject

environments/tests/env-test-minimum-deps.yml

@@ -22,4 +22,5 @@ dependencies:
   - scikit-learn==0.23.*
   - scipy==1.5.*
   - shapely==2.0.*
+  - tomli
   - typeguard

environments/tests/requirements-test-latest-deps.txt

@@ -17,4 +17,5 @@ rio-vrt>=0.3
 scikit-learn>=0.23
 scipy>=1.5
 shapely>=2.0
+tomli
 typeguard

osmnx/_http.py

@@ -55,6 +55,9 @@ def _save_to_cache(
         if not ok:  # pragma: no cover
             msg = "Did not save to cache because HTTP status code is not OK"
             utils.log(msg, level=lg.WARNING)
+        elif isinstance(response_json, dict) and ("remark" in response_json):  # pragma: no cover
+            msg = f"Did not save to cache because response contains remark: {response_json['remark']!r}"
+            utils.log(msg, lg.WARNING)
         else:
             # create the folder on the disk if it doesn't already exist
             cache_folder = Path(settings.cache_folder)
@@ -118,19 +121,8 @@ def _retrieve_from_cache(url: str) -> dict[str, Any] | list[dict[str, Any]] | No
         # return cached response for this url if exists, otherwise return None
         cache_filepath = _url_in_cache(url)
         if cache_filepath is not None:
-            response_json: dict[str, Any] | list[dict[str, Any]] = json.loads(
-                cache_filepath.read_text(encoding="utf-8"),
-            )
-
-            # return None if there is a server remark in the cached response
-            if isinstance(response_json, dict) and ("remark" in response_json):  # pragma: no cover
-                msg = (
-                    f"Ignoring cache file {str(cache_filepath)!r} because "
-                    f"it contains a remark: {response_json['remark']!r}"
-                )
-                utils.log(msg, lg.WARNING)
-                return None
-
+            response_json: dict[str, Any] | list[dict[str, Any]]
+            response_json = json.loads(cache_filepath.read_text(encoding="utf-8"))
             msg = f"Retrieved response from cache file {str(cache_filepath)!r}"
             utils.log(msg, lg.INFO)
             return response_json

osmnx/_nominatim.py

@@ -145,12 +145,7 @@ def _nominatim_request(
         )
         utils.log(msg, level=lg.WARNING)
         time.sleep(error_pause)
-        return _nominatim_request(
-            params,
-            request_type=request_type,
-            pause=pause,
-            error_pause=error_pause,
-        )
+        return _nominatim_request(params, request_type=request_type)
 
     response_json = _http._parse_response(response)
     if not isinstance(response_json, list):

osmnx/_overpass.py

@@ -141,8 +141,8 @@ def _get_network_filter(network_type: str) -> str:
 def _get_overpass_pause(
     base_endpoint: str,
     *,
-    recursive_delay: float = 5,
-    default_duration: float = 60,
+    recursive_delay: float = 4,
+    default_duration: float = 50,
 ) -> float:
     """
     Retrieve a pause duration from the Overpass API status endpoint.
@@ -183,7 +183,7 @@ def _get_overpass_pause(
         response_text = response.text
     except RequestsConnectionError as e:  # pragma: no cover
         # cannot reach status endpoint: log error and return default duration
-        msg = f"Unable to query {url}, {e}"
+        msg = f"Unable to reach {url}, {e}"
         utils.log(msg, level=lg.ERROR)
         return default_duration
 
@@ -209,7 +209,7 @@ def _get_overpass_pause(
         if status_first_part == "Slot":
             utc_time_str = status.split(" ")[3]
             pattern = "%Y-%m-%dT%H:%M:%SZ,"
-            utc_time = dt.datetime.strptime(utc_time_str, pattern).astimezone(dt.timezone.utc)
+            utc_time = dt.datetime.strptime(utc_time_str, pattern).replace(tzinfo=dt.timezone.utc)
             utc_now = dt.datetime.now(tz=dt.timezone.utc)
             seconds = int(np.ceil((utc_time - utc_now).total_seconds()))
             pause = max(seconds, 1)
@@ -432,7 +432,7 @@ def _overpass_request(
     data: OrderedDict[str, Any],
     *,
     pause: float | None = None,
-    error_pause: float = 60,
+    error_pause: float = 55,
 ) -> dict[str, Any]:
     """
     Send a HTTP POST request to the Overpass API and return response.
@@ -465,11 +465,11 @@ def _overpass_request(
 
     # pause then request this URL
     if pause is None:
-        this_pause = _get_overpass_pause(settings.overpass_url)
+        pause = _get_overpass_pause(settings.overpass_url)
     hostname = _http._hostname_from_url(url)
-    msg = f"Pausing {this_pause} second(s) before making HTTP POST request to {hostname!r}"
+    msg = f"Pausing {pause} second(s) before making HTTP POST request to {hostname!r}"
     utils.log(msg, level=lg.INFO)
-    time.sleep(this_pause)
+    time.sleep(pause)
 
     # transmit the HTTP POST request
     msg = f"Post {prepared_url} with timeout={settings.requests_timeout}"
@@ -484,14 +484,13 @@ def _overpass_request(
 
     # handle 429 and 504 errors by pausing then recursively re-trying request
     if response.status_code in {429, 504}:  # pragma: no cover
-        this_pause = error_pause + _get_overpass_pause(settings.overpass_url)
         msg = (
             f"{hostname!r} responded {response.status_code} {response.reason}: "
-            f"we'll retry in {this_pause} secs"
+            f"we'll retry in {error_pause} secs"
         )
         utils.log(msg, level=lg.WARNING)
-        time.sleep(this_pause)
-        return _overpass_request(data, pause=pause, error_pause=error_pause)
+        time.sleep(error_pause)
+        return _overpass_request(data)
 
     response_json = _http._parse_response(response)
     if not isinstance(response_json, dict):  # pragma: no cover

osmnx/_version.py

@@ -1,3 +1,3 @@
 """OSMnx package version information."""
 
-__version__ = "2.0.1"
+__version__ = "2.0.2dev"