Merge remote-tracking branch 'upstream/main' into replace_napolean_wi…

…th_numpydoc * upstream/main: (39 commits) Bump scitools/workflows from 2024.03.3 to 2024.04.0 (SciTools#5907) [pre-commit.ci] pre-commit autoupdate (SciTools#5906) Updated environment lockfiles (SciTools#5904) Ignore flaticon.com in linkchecks. (SciTools#5905) Implement lazy area weights (SciTools#5658) Add option to specify chunks in `iris.util.broadcast_to_shape` (SciTools#5620) Unpin sphinx (SciTools#5901) DOC: clarify save_pairs_from_cube docstring (SciTools#5783) Restore latest Whats New files. Whats new updates for `v3.9.0rc0` (SciTools#5899) nep29: drop py39 and support py312 (SciTools#5894) Support NetCDF v3 files in chunking control code. (SciTools#5897) Avoid computing lazy scalar coordinates when printing a Cube (v2) (SciTools#5896) Force pytest colour output on GitHub Actions (SciTools#5895) Make typing 3.9 compatible. Improve typing readability. Updated environment lockfiles (SciTools#5892) [pre-commit.ci] pre-commit autoupdate What's New entry for SciTools#5740 . Iris to GeoVista conversion (SciTools#5740) ...
tkknight · Apr 10, 2024 · aa4e24b · aa4e24b
2 parents 3f08516 + e26bfaf
commit aa4e24b
Show file tree

Hide file tree

Showing 136 changed files with 2,268 additions and 1,442 deletions.
diff --git a/.github/workflows/ci-manifest.yml b/.github/workflows/ci-manifest.yml
@@ -23,4 +23,4 @@ concurrency:
 jobs:
   manifest:
     name: "check-manifest"
-    uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.03.1
+    uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.04.0
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -35,18 +35,18 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.11"]
+        python-version: ["3.12"]
         session: ["doctest", "gallery", "linkcheck"]
         include:
           - os: "ubuntu-latest"
-            python-version: "3.11"
+            python-version: "3.12"
             session: "tests"
             coverage: "--coverage"
           - os: "ubuntu-latest"
-            python-version: "3.10"
+            python-version: "3.11"
             session: "tests"
           - os: "ubuntu-latest"
-            python-version: "3.9"
+            python-version: "3.10"
             session: "tests"
 
     env:
@@ -136,6 +136,8 @@ jobs:
       - name: "iris ${{ matrix.session }}"
         env:
           PY_VER: ${{ matrix.python-version }}
+          # Force coloured output on GitHub Actions.
+          PY_COLORS: "1"
         run: |
           nox --session ${{ matrix.session }} -- --verbose ${{ matrix.coverage }}
 

diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml
@@ -52,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         session: ["wheel"]
     env:
       ENV_NAME: "ci-wheels"

diff --git a/.github/workflows/refresh-lockfiles.yml b/.github/workflows/refresh-lockfiles.yml
@@ -14,5 +14,5 @@ on:
 
 jobs:
   refresh_lockfiles:
-    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.03.1
+    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.04.0
     secrets: inherit
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ minimum_pre_commit_version: 1.21.0
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
         # Prevent giant files from being committed.
     -   id: check-added-large-files
@@ -29,7 +29,7 @@ repos:
     -   id: no-commit-to-branch
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.2.1"
+    rev: "v0.3.5"
     hooks:
     -   id: ruff
         types: [file, python]
@@ -63,7 +63,7 @@ repos:
         types: [file, python]
 
 -   repo: https://github.com/numpy/numpydoc
-    rev: v1.6.0
+    rev: v1.7.0
     hooks:
       - id: numpydoc-validation
         exclude: "^lib/iris/tests/|docs/gallery_code/"

diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json
@@ -23,7 +23,7 @@
     //  * No build-time environment variables.
     //  * Is run in the same environment as the ASV install itself.
     "delegated_env_commands": [
-        "PY_VER=3.11 nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose"
+        "PY_VER=3.12 nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose"
     ],
     // The parent directory of the above environment.
     //  The most recently modified environment in the directory will be used.

diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Common code for benchmarks."""
+
 from os import environ
 import resource
 

diff --git a/benchmarks/benchmarks/cperf/__init__.py b/benchmarks/benchmarks/cperf/__init__.py
@@ -9,6 +9,7 @@
 Files available from the UK Met Office:
   moo ls moose:/adhoc/projects/avd/asv/data_for_nightly_tests/
 """
+
 import numpy as np
 
 from iris import load_cube

diff --git a/benchmarks/benchmarks/cperf/equality.py b/benchmarks/benchmarks/cperf/equality.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Equality benchmarks for the CPerf scheme of the UK Met Office's NG-VAT project."""
+
 from .. import on_demand_benchmark
 from . import SingleDiagnosticMixin
 

diff --git a/benchmarks/benchmarks/cperf/load.py b/benchmarks/benchmarks/cperf/load.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """File loading benchmarks for the CPerf scheme of the UK Met Office's NG-VAT project."""
+
 from .. import on_demand_benchmark
 from . import SingleDiagnosticMixin
 

diff --git a/benchmarks/benchmarks/experimental/ugrid/regions_combine.py b/benchmarks/benchmarks/experimental/ugrid/regions_combine.py
@@ -16,6 +16,7 @@
   run-time that scale with data size.
 
 """
+
 import os
 
 import dask.array as da

diff --git a/benchmarks/benchmarks/generate_data/__init__.py b/benchmarks/benchmarks/generate_data/__init__.py
@@ -14,6 +14,7 @@
 benchmark sequence runs over two different Python versions.
 
 """
+
 from contextlib import contextmanager
 from inspect import getsource
 from os import environ

diff --git a/benchmarks/benchmarks/generate_data/ugrid.py b/benchmarks/benchmarks/generate_data/ugrid.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Scripts for generating supporting data for UGRID-related benchmarking."""
+
 from iris import load_cube as iris_loadcube
 from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD
 

diff --git a/benchmarks/benchmarks/iterate.py b/benchmarks/benchmarks/iterate.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Iterate benchmark tests."""
+
 import numpy as np
 
 from iris import coords, cube, iterate

diff --git a/benchmarks/benchmarks/plot.py b/benchmarks/benchmarks/plot.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Plot benchmark tests."""
+
 import matplotlib as mpl
 import numpy as np
 

diff --git a/benchmarks/benchmarks/save.py b/benchmarks/benchmarks/save.py
@@ -11,6 +11,7 @@
            run-time that scale with data size.
 
 """
+
 from iris import save
 from iris.experimental.ugrid import save_mesh
 

diff --git a/benchmarks/benchmarks/sperf/__init__.py b/benchmarks/benchmarks/sperf/__init__.py
@@ -7,6 +7,7 @@
 SPerf = assessing performance against a series of increasingly large LFRic
 datasets.
 """
+
 from iris import load_cube
 
 # TODO: remove uses of PARSE_UGRID_ON_LOAD once UGRID parsing is core behaviour.

diff --git a/benchmarks/benchmarks/sperf/combine_regions.py b/benchmarks/benchmarks/sperf/combine_regions.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Region combine benchmarks for the SPerf scheme of the UK Met Office's NG-VAT project."""
+
 import os.path
 
 from dask import array as da

diff --git a/benchmarks/benchmarks/sperf/equality.py b/benchmarks/benchmarks/sperf/equality.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """Equality benchmarks for the SPerf scheme of the UK Met Office's NG-VAT project."""
+
 from .. import on_demand_benchmark
 from . import FileMixin
 

diff --git a/benchmarks/benchmarks/sperf/load.py b/benchmarks/benchmarks/sperf/load.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """File loading benchmarks for the SPerf scheme of the UK Met Office's NG-VAT project."""
+
 from .. import on_demand_benchmark
 from . import FileMixin
 

diff --git a/benchmarks/benchmarks/sperf/save.py b/benchmarks/benchmarks/sperf/save.py
@@ -3,6 +3,7 @@
 # This file is part of Iris and is released under the BSD license.
 # See LICENSE in the root of the repository for full licensing details.
 """File saving benchmarks for the SPerf scheme of the UK Met Office's NG-VAT project."""
+
 import os.path
 
 from iris import save

diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py
@@ -67,7 +67,7 @@ def _check_requirements(package: str) -> None:
 
 def _prep_data_gen_env() -> None:
     """Create or access a separate, unchanging environment for generating test data."""
-    python_version = "3.11"
+    python_version = "3.12"
     data_gen_var = "DATA_GEN_PYTHON"
     if data_gen_var in environ:
         echo("Using existing data generation environment.")

diff --git a/docs/gallery_code/oceanography/plot_load_nemo.py b/docs/gallery_code/oceanography/plot_load_nemo.py
@@ -45,11 +45,11 @@ def main():
 
     # Include the point's position in the plot's title
     lat_point = cube.coord("latitude").points[y_point_index, x_point_index]
-    lat_string = "{:.3f}\u00B0 {}".format(
+    lat_string = "{:.3f}\u00b0 {}".format(
         abs(lat_point), "N" if lat_point > 0.0 else "S"
     )
     lon_point = cube.coord("longitude").points[y_point_index, x_point_index]
-    lon_string = "{:.3f}\u00B0 {}".format(
+    lon_string = "{:.3f}\u00b0 {}".format(
         abs(lon_point), "E" if lon_point > 0.0 else "W"
     )
     plt.title("{} at {} {}".format(cube.long_name.capitalize(), lat_string, lon_string))

diff --git a/docs/src/conf.py b/docs/src/conf.py
@@ -272,6 +272,8 @@ def _dotv(version):
     "pandas": ("https://pandas.pydata.org/docs/", None),
     "dask": ("https://docs.dask.org/en/stable/", None),
     "xarray": ("https://docs.xarray.dev/en/stable/", None),
+    "geovista": ("https://geovista.readthedocs.io/en/latest/", None),
+    "pyvista": ("https://docs.pyvista.org/", None),
 }
 
 # The name of the Pygments (syntax highlighting) style to use.
@@ -397,16 +399,6 @@ def _dotv(version):
 html_static_path = ["_static"]
 html_style = "theme_override.css"
 
-# this allows for using datatables: https://datatables.net/.
-# the version can be manually upgraded by changing the urls below.
-html_css_files = [
-    "https://cdn.datatables.net/1.13.4/css/jquery.dataTables.min.css",
-]
-
-html_js_files = [
-    "https://cdn.datatables.net/1.13.4/js/jquery.dataTables.min.js",
-]
-
 # url link checker.  Some links work but report as broken, lets ignore them.
 # See https://www.sphinx-doc.org/en/1.2/config.html#options-for-the-linkcheck-builder
 linkcheck_ignore = [
@@ -428,6 +420,7 @@ def _dotv(version):
     "https://stickler-ci.com/",
     "https://twitter.com/scitools_iris",
     "https://stackoverflow.com/questions/tagged/python-iris",
+    "https://www.flaticon.com/",
 ]
 
 # list of sources to exclude from the build.

diff --git a/docs/src/further_topics/missing_data_handling.rst b/docs/src/further_topics/missing_data_handling.rst
@@ -22,6 +22,7 @@ On load, any fill-value or missing data value defined in the loaded dataset
 should be used as the ``fill_value`` of the NumPy masked array data attribute of the
 :class:`~iris.cube.Cube`. This will only appear when the cube's data is realised.
 
+.. _missing_data_saving:
 
 Saving
 ------
@@ -37,7 +38,8 @@ For example::
 .. note::
     Not all savers accept the ``fill_value`` keyword argument.
 
-Iris will check for and issue warnings of fill-value 'collisions'.
+Iris will check for and issue warnings of fill-value 'collisions' (exception:
+**NetCDF**, see the heading below).
 This basically means that whenever there are unmasked values that would read back
 as masked, we issue a warning and suggest a workaround.
 
@@ -51,6 +53,8 @@ This will occur in the following cases:
 NetCDF
 ~~~~~~
 
+:term:`NetCDF Format`
+
 NetCDF is a special case, because all ordinary variable data is "potentially masked",
 owing to the use of default fill values. The default fill-value used depends on the type
 of the variable data.
@@ -64,6 +68,16 @@ The exceptions to this are:
 * Small integers create problems by *not* having the exemption applied to byte data.
   Thus, in principle, ``int32`` data cannot use the full range of 2**16 valid values.
 
+Warnings are not issued for NetCDF fill value collisions. Increasingly large
+and complex parallel I/O operations unfortunately made this feature
+un-maintainable and it was retired in Iris 3.9 (:pull:`5833`).
+
+If you need to know about collisions then you can perform your own checks ahead
+of saving. Such operations can be run lazily (:term:`Lazy Data`). Here is an
+example::
+
+    >>> default_fill = netCDF4.default_fillvals[my_cube.dtype.str[1:]]
+    >>> fill_present = (my_cube.lazy_data() == default_fill).any().compute()
 
 Merging
 -------

diff --git a/docs/src/further_topics/ugrid/images/plotting.png b/docs/src/further_topics/ugrid/images/plotting.png
diff --git a/docs/src/further_topics/ugrid/images/plotting_basic.png b/docs/src/further_topics/ugrid/images/plotting_basic.png
diff --git a/docs/src/further_topics/ugrid/images/plotting_global.png b/docs/src/further_topics/ugrid/images/plotting_global.png
-Original file line number
+Diff line change
@@ Expand Up / @@ -16,6 +16,7 @@ @@
       run-time that scale with data size.
     """
     import os
     import dask.array as da
@@ Expand Down @@