Test failures with NetCDF 4.9.0

[sebastic]

As reported in Debian Bug #1013754, the h5netcdf testsuite fails with NetCDF 4.9.0 which added more compression options:

=================================== FAILURES ===================================
______________________ test_write_legacyapi_read_netCDF4 _______________________

tmp_local_netcdf = '/tmp/pytest-of-debci/pytest-0/test_write_legacyapi_read_netC0/testfile.nc'

    def test_write_legacyapi_read_netCDF4(tmp_local_netcdf):
>       roundtrip_legacy_netcdf(tmp_local_netcdf, netCDF4, legacyapi)

tests/test_h5netcdf.py:450: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_h5netcdf.py:446: in roundtrip_legacy_netcdf
    read_legacy_netcdf(tmp_netcdf, read_module, write_module)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

tmp_netcdf = '/tmp/pytest-of-debci/pytest-0/test_write_legacyapi_read_netC0/testfile.nc'
read_module = <module 'netCDF4' from '/usr/lib/python3/dist-packages/netCDF4/__init__.py'>
write_module = <module 'h5netcdf.legacyapi' from '/usr/lib/python3/dist-packages/h5netcdf/legacyapi.py'>

    def read_legacy_netcdf(tmp_netcdf, read_module, write_module):
        ds = read_module.Dataset(tmp_netcdf, "r")
        assert ds.ncattrs() == ["global", "other_attr"]
        assert ds.getncattr("global") == 42
        if write_module is not netCDF4:
            # skip for now: https://github.com/Unidata/netcdf4-python/issues/388
            assert ds.other_attr == "yes"
        with pytest.raises(AttributeError):
            ds.does_not_exist
        assert set(ds.dimensions) == set(
            ["x", "y", "z", "empty", "string3", "mismatched_dim", "unlimited"]
        )
        assert set(ds.variables) == set(
            [
                "foo",
                "y",
                "z",
                "intscalar",
                "scalar",
                "var_len_str",
                "mismatched_dim",
                "foo_unlimited",
            ]
        )
    
        assert set(ds.groups) == set(["subgroup"])
        assert ds.parent is None
        v = ds.variables["foo"]
        assert array_equal(v, np.ones((4, 5)))
        assert v.dtype == float
        assert v.dimensions == ("x", "y")
        assert v.ndim == 2
        assert v.ncattrs() == ["units"]
        if write_module is not netCDF4:
            assert v.getncattr("units") == "meters"
        assert tuple(v.chunking()) == (4, 5)
>       assert v.filters() == {
            "complevel": 4,
            "fletcher32": False,
            "shuffle": True,
            "zlib": True,
        }
E       AssertionError: assert {'blosc': Fal...': False, ...} == {'complevel':... 'zlib': True}
E         Omitting 4 identical items, use -vv to show
E         Left contains 4 more items:
E         {'blosc': False, 'bzip2': False, 'szip': False, 'zstd': False}
E         Use -v to get the full diff

tests/test_h5netcdf.py:259: AssertionError
_______________________ test_write_h5netcdf_read_netCDF4 _______________________

tmp_local_netcdf = '/tmp/pytest-of-debci/pytest-0/test_write_h5netcdf_read_netCD0/testfile.nc'

    def test_write_h5netcdf_read_netCDF4(tmp_local_netcdf):
        write_h5netcdf(tmp_local_netcdf)
>       read_legacy_netcdf(tmp_local_netcdf, netCDF4, h5netcdf)

tests/test_h5netcdf.py:468: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

tmp_netcdf = '/tmp/pytest-of-debci/pytest-0/test_write_h5netcdf_read_netCD0/testfile.nc'
read_module = <module 'netCDF4' from '/usr/lib/python3/dist-packages/netCDF4/__init__.py'>
write_module = <module 'h5netcdf' from '/usr/lib/python3/dist-packages/h5netcdf/__init__.py'>

    def read_legacy_netcdf(tmp_netcdf, read_module, write_module):
        ds = read_module.Dataset(tmp_netcdf, "r")
        assert ds.ncattrs() == ["global", "other_attr"]
        assert ds.getncattr("global") == 42
        if write_module is not netCDF4:
            # skip for now: https://github.com/Unidata/netcdf4-python/issues/388
            assert ds.other_attr == "yes"
        with pytest.raises(AttributeError):
            ds.does_not_exist
        assert set(ds.dimensions) == set(
            ["x", "y", "z", "empty", "string3", "mismatched_dim", "unlimited"]
        )
        assert set(ds.variables) == set(
            [
                "foo",
                "y",
                "z",
                "intscalar",
                "scalar",
                "var_len_str",
                "mismatched_dim",
                "foo_unlimited",
            ]
        )
    
        assert set(ds.groups) == set(["subgroup"])
        assert ds.parent is None
        v = ds.variables["foo"]
        assert array_equal(v, np.ones((4, 5)))
        assert v.dtype == float
        assert v.dimensions == ("x", "y")
        assert v.ndim == 2
        assert v.ncattrs() == ["units"]
        if write_module is not netCDF4:
            assert v.getncattr("units") == "meters"
        assert tuple(v.chunking()) == (4, 5)
>       assert v.filters() == {
            "complevel": 4,
            "fletcher32": False,
            "shuffle": True,
            "zlib": True,
        }
E       AssertionError: assert {'blosc': Fal...': False, ...} == {'complevel':... 'zlib': True}
E         Omitting 4 identical items, use -vv to show
E         Left contains 4 more items:
E         {'blosc': False, 'bzip2': False, 'szip': False, 'zstd': False}
E         Use -v to get the full diff

tests/test_h5netcdf.py:259: AssertionError
=========================== short test summary info ============================
FAILED tests/test_h5netcdf.py::test_write_legacyapi_read_netCDF4 - AssertionE...
FAILED tests/test_h5netcdf.py::test_write_h5netcdf_read_netCDF4 - AssertionEr...
=================== 2 failed, 69 passed, 22 skipped in 2.44s ===================

This patch resolves the issue:

--- a/h5netcdf/tests/test_h5netcdf.py
+++ b/h5netcdf/tests/test_h5netcdf.py
@@ -256,12 +256,11 @@ def read_legacy_netcdf(tmp_netcdf, read_
     if write_module is not netCDF4:
         assert v.getncattr("units") == "meters"
     assert tuple(v.chunking()) == (4, 5)
-    assert v.filters() == {
-        "complevel": 4,
-        "fletcher32": False,
-        "shuffle": True,
-        "zlib": True,
-    }
+    filters = v.filters()
+    assert filters['complevel'] == 4
+    assert filters['fletcher32'] is False
+    assert filters['shuffle'] is True
+    assert filters['zlib'] is True
 
     v = ds.variables["y"]
     assert array_equal(v, np.r_[np.arange(4), [-1]])
@@ -271,12 +270,11 @@ def read_legacy_netcdf(tmp_netcdf, read_
     assert v.ncattrs() == ["_FillValue"]
     assert v.getncattr("_FillValue") == -1
     assert v.chunking() == "contiguous"
-    assert v.filters() == {
-        "complevel": 0,
-        "fletcher32": False,
-        "shuffle": False,
-        "zlib": False,
-    }
+    filters = v.filters()
+    assert filters['complevel'] == 0
+    assert filters['fletcher32'] is False
+    assert filters['shuffle'] is False
+    assert filters['zlib'] is False
     ds.close()
 
     # Check the behavior if h5py. Cannot expect h5netcdf to overcome these

[kmuehlbauer]

@sebastic Thanks for raising and the attached patch. Patchrelease will follow shortly.

[kmuehlbauer]

Thanks again @sebastic! h5netcdf patch release 1.0.1 is available on PyPI now. I'll leave this issue open until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1013754 is resolved (just in case there are still problems over at debian).

[kmuehlbauer]

We can't test this with our current CI workflow as netCDF 4.9.0 is not yet available on conda-forge, see conda-forge/libnetcdf-feedstock#140. So for now relying on the debian tests, that the patch works.

[sebastic]

The package built successfully:

https://buildd.debian.org/status/package.php?p=python-h5netcdf

And the autopkgtests are looking good so far:

https://qa.debian.org/excuses.php?package=python-h5netcdf

[kmuehlbauer]

Thanks @sebastic that's good news.

[sebastic]

The autopkgtests passed on all architectures, and the package has migrated to testing.