Merge pull request pangeo-forge#216 from TomAugspurger/fix/214-key-error

finalize_target handles coordinateless dimensions

pangeo_forge_recipes/recipes/xarray_zarr.py

@@ -641,7 +641,10 @@ def finalize_target(
         logger.info("Consolidating dimension coordinate arrays")
         target_mapper = target.get_mapper()
         group = zarr.open(target_mapper, mode="a")
-        dims = _gather_coordinate_dimensions(group)
+        # https://github.com/pangeo-forge/pangeo-forge-recipes/issues/214
+        # intersect the dims from the array metadata with the Zarr group
+        # to handle coordinateless dimensions.
+        dims = set(_gather_coordinate_dimensions(group)) & set(group)
         for dim in dims:
             arr = group[dim]
             attrs = dict(arr.attrs)

tests/conftest.py

@@ -211,6 +211,18 @@ def daily_xarray_dataset():
     return ds
 
 
+@pytest.fixture(scope="session")
+def daily_xarray_dataset_with_coordinateless_dimension(daily_xarray_dataset):
+    """
+    A Dataset with a coordinateless dimension.
+
+    Reproduces https://github.com/pangeo-forge/pangeo-forge-recipes/issues/214
+    """
+    ds = daily_xarray_dataset.copy()
+    del ds["lon"]
+    return ds
+
+
 @pytest.fixture(scope="session")
 def netcdf_local_paths_sequential_1d(daily_xarray_dataset, tmpdir_factory):
     return make_netcdf_local_paths(daily_xarray_dataset, tmpdir_factory, "D", split_up_files_by_day)
@@ -259,6 +271,18 @@ def netcdf_local_paths_sequential_multivariable(request):
     return request.param
 
 
+@pytest.fixture(scope="session",)
+def netcdf_local_paths_sequential_multivariable_with_coordinateless_dimension(
+    daily_xarray_dataset_with_coordinateless_dimension, tmpdir_factory
+):
+    return make_netcdf_local_paths(
+        daily_xarray_dataset_with_coordinateless_dimension,
+        tmpdir_factory,
+        "D",
+        split_up_files_by_variable_and_day,
+    )
+
+
 @pytest.fixture(
     scope="session",
     params=[
@@ -297,6 +321,18 @@ def netcdf_http_paths_sequential_1d(request):
     return request.param
 
 
+@pytest.fixture(scope="session")
+def netcdf_local_paths_sequential_with_coordinateless_dimension(
+    daily_xarray_dataset_with_coordinateless_dimension, tmpdir_factory
+):
+    return make_netcdf_local_paths(
+        daily_xarray_dataset_with_coordinateless_dimension,
+        tmpdir_factory,
+        "D",
+        split_up_files_by_day,
+    )
+
+
 # FilePattern fixtures ----------------------------------------------------------------------------
 
 
@@ -328,6 +364,16 @@ def netcdf_http_file_pattern_sequential_1d(netcdf_http_paths_sequential_1d):
     return make_file_pattern(netcdf_http_paths_sequential_1d)
 
 
+@pytest.fixture(scope="session")
+def netcdf_local_file_pattern_sequential_with_coordinateless_dimension(
+    netcdf_local_paths_sequential_with_coordinateless_dimension,
+):
+    """
+    Filepattern where one of the dimensions doesn't have a coordinate.
+    """
+    return make_file_pattern(netcdf_local_paths_sequential_with_coordinateless_dimension)
+
+
 # Storage fixtures --------------------------------------------------------------------------------
 
 

tests/recipe_tests/test_XarrayZarrRecipe.py

@@ -358,6 +358,26 @@ def test_no_consolidate_dimension_coordinates(netCDFtoZarr_recipe):
     assert store["time"].chunks == (file_pattern.nitems_per_input["time"],)
 
 
+def test_consolidate_dimension_coordinates_with_coordinateless_dimension(
+    netcdf_local_file_pattern_sequential_with_coordinateless_dimension,
+    daily_xarray_dataset_with_coordinateless_dimension,
+    tmp_target,
+    tmp_cache,
+    tmp_metadata_target,
+):
+    RecipeClass, file_pattern, kwargs, ds_expected, target = make_netCDFtoZarr_recipe(
+        netcdf_local_file_pattern_sequential_with_coordinateless_dimension,
+        daily_xarray_dataset_with_coordinateless_dimension,
+        tmp_target,
+        tmp_cache,
+        tmp_metadata_target,
+    )
+    rec = RecipeClass(file_pattern, **kwargs)
+    rec.to_function()()
+    ds_actual = xr.open_zarr(target.get_mapper()).load()
+    xr.testing.assert_identical(ds_actual, ds_expected)
+
+
 def test_lock_timeout(netCDFtoZarr_recipe_sequential_only, execute_recipe_no_dask):
     RecipeClass, file_pattern, kwargs, ds_expected, target = netCDFtoZarr_recipe_sequential_only