Merge #321

321: Convert `byte_no_cf` and `cf_nasa_4326` to Zarr r=metasim,jdroenner a=lnicola - [x] I agree to follow the project's [code of conduct](https://github.com/georust/gdal/blob/master/CODE_OF_CONDUCT.md). - [ ] I added an entry to `CHANGES.md` if knowledge of this change could be valuable to users. --- Closes #239 I left `alldatatypes.nc` untouched because it's string arrays are not supported. Unfortunately, I had to disable these tests on pre-3.4 GDAL, where Zarr is not supported. Co-authored-by: Laurențiu Nicola <[email protected]>
georust · Oct 21, 2022 · c00ab44 · c00ab44
2 parents 7079683 + a61bf52
commit c00ab44
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 40 deletions.
diff --git a/fixtures/byte_no_cf.nc b/fixtures/byte_no_cf.nc
diff --git a/fixtures/byte_no_cf.zarr.zip b/fixtures/byte_no_cf.zarr.zip
diff --git a/fixtures/cf_nasa_4326.nc b/fixtures/cf_nasa_4326.nc
diff --git a/fixtures/cf_nasa_4326.zarr.zip b/fixtures/cf_nasa_4326.zarr.zip
diff --git a/src/programs/raster/mdimtranslate.rs b/src/programs/raster/mdimtranslate.rs
@@ -230,11 +230,12 @@ mod tests {
 
     use super::*;
 
-    use crate::{test_utils::TempFixture, DatasetOptions, Driver, GdalOpenFlags};
+    use crate::{DatasetOptions, Driver, GdalOpenFlags};
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_build_tiff_from_path() {
-        let fixture = TempFixture::fixture("cf_nasa_4326.nc");
+        let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -243,7 +244,6 @@ mod tests {
             sibling_files: None,
         };
         let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
-
         let mem_file_path = "/vsimem/2d3e9124-a7a0-413e-97b5-e79d46e50ff8";
 
         let dataset = multi_dim_translate(
@@ -265,8 +265,9 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_build_tiff_from_dataset() {
-        let fixture = TempFixture::fixture("cf_nasa_4326.nc");
+        let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,

diff --git a/src/raster/mdarray.rs b/src/raster/mdarray.rs
@@ -803,8 +803,9 @@ mod tests {
     use crate::{test_utils::TempFixture, Dataset, DatasetOptions, GdalOpenFlags};
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_root_group_name() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -819,8 +820,9 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_array_names() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -830,14 +832,18 @@ mod tests {
         };
         let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
         let root_group = dataset.root_group().unwrap();
-        let options = CslStringList::new(); //Driver specific options determining how groups should be retrieved. Pass nullptr for default behavior.
+        let options = CslStringList::new();
         let array_names = root_group.array_names(options);
-        assert_eq!(array_names, vec!["Band1".to_string()])
+        assert_eq!(
+            array_names,
+            vec!["X".to_string(), "Y".to_string(), "byte_no_cf".to_string()]
+        )
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_n_dimension() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -847,16 +853,17 @@ mod tests {
         };
         let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
         let root_group = dataset.root_group().unwrap();
-        let array_name = "Band1".to_string();
-        let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
+        let array_name = "byte_no_cf".to_string();
+        let options = CslStringList::new();
         let md_array = root_group.open_md_array(&array_name, options).unwrap();
         let n_dimension = md_array.num_dimensions();
         assert_eq!(2, n_dimension);
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_n_elements() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -866,16 +873,17 @@ mod tests {
         };
         let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
         let root_group = dataset.root_group().unwrap();
-        let array_name = "Band1".to_string();
-        let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
+        let array_name = "byte_no_cf".to_string();
+        let options = CslStringList::new();
         let md_array = root_group.open_md_array(&array_name, options).unwrap();
         let n_elements = md_array.num_elements();
         assert_eq!(400, n_elements);
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_dimension_name() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -892,24 +900,25 @@ mod tests {
             .into_iter()
             .map(|dimensions| dimensions.name())
             .collect();
-        assert_eq!(group_dimensions_names, vec!["x", "y"]);
+        assert_eq!(group_dimensions_names, vec!["X", "Y"]);
 
         // array dimensions
 
-        let array_name = "Band1".to_string();
-        let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
+        let array_name = "byte_no_cf".to_string();
+        let options = CslStringList::new();
         let md_array = root_group.open_md_array(&array_name, options).unwrap();
         let dimensions = md_array.dimensions().unwrap();
         let mut dimension_names = Vec::new();
         for dimension in dimensions {
             dimension_names.push(dimension.name());
         }
-        assert_eq!(dimension_names, vec!["y".to_string(), "x".to_string()])
+        assert_eq!(dimension_names, vec!["Y".to_string(), "X".to_string()])
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_dimension_size() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -919,8 +928,8 @@ mod tests {
         };
         let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
         let root_group = dataset.root_group().unwrap();
-        let array_name = "Band1".to_string();
-        let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
+        let array_name = "byte_no_cf".to_string();
+        let options = CslStringList::new();
         let md_array = root_group.open_md_array(&array_name, options).unwrap();
         let dimensions = md_array.dimensions().unwrap();
         let mut dimensions_size = Vec::new();
@@ -931,8 +940,9 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_read_data() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -944,7 +954,7 @@ mod tests {
 
         let root_group = dataset.root_group().unwrap();
         let md_array = root_group
-            .open_md_array("Band1", CslStringList::new())
+            .open_md_array("byte_no_cf", CslStringList::new())
             .unwrap();
 
         let values = md_array.read_as::<u8>(vec![0, 0], vec![20, 20]).unwrap();
@@ -956,7 +966,10 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_read_string_array() {
+        // Beware https://github.com/georust/gdal/issues/299 if you want to reuse this
+        // This can't be Zarr because it doesn't support string arrays
         let fixture = TempFixture::fixture("alldatatypes.nc");
 
         let dataset_options = DatasetOptions {
@@ -986,8 +999,9 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_datatype() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -1000,7 +1014,7 @@ mod tests {
         let root_group = dataset.root_group().unwrap();
 
         let md_array = root_group
-            .open_md_array("Band1", CslStringList::new())
+            .open_md_array("byte_no_cf", CslStringList::new())
             .unwrap();
 
         let datatype = md_array.datatype();
@@ -1011,8 +1025,9 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_spatial_ref() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -1024,7 +1039,7 @@ mod tests {
 
         let root_group = dataset.root_group().unwrap();
         let md_array = root_group
-            .open_md_array("Band1", CslStringList::new())
+            .open_md_array("byte_no_cf", CslStringList::new())
             .unwrap();
 
         let spatial_ref = md_array.spatial_reference().unwrap();
@@ -1035,8 +1050,9 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_no_data_value() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -1048,15 +1064,16 @@ mod tests {
 
         let root_group = dataset.root_group().unwrap();
         let md_array = root_group
-            .open_md_array("Band1", CslStringList::new())
+            .open_md_array("byte_no_cf", CslStringList::new())
             .unwrap();
 
         assert_eq!(md_array.no_data_value_as_double(), Some(0.));
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_attributes() {
-        let fixture = TempFixture::fixture("cf_nasa_4326.nc");
+        let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -1101,15 +1118,13 @@ mod tests {
             "air_temperature"
         );
 
-        assert_eq!(
-            md_array.attribute("_FillValue").unwrap().read_as_f64(),
-            -9999.
-        );
+        assert_eq!(md_array.no_data_value_as_double().unwrap(), -9999.);
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_unit() {
-        let fixture = TempFixture::fixture("cf_nasa_4326.nc");
+        let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
@@ -1147,19 +1162,25 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
     fn test_stats() {
-        let fixture = TempFixture::fixture("byte_no_cf.nc");
+        // make a copy to avoid writing the statistics into the original file
+        let fixture = TempFixture::fixture("byte_no_cf.zarr.zip");
 
         let dataset_options = DatasetOptions {
             open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
             allowed_drivers: None,
             open_options: None,
             sibling_files: None,
         };
-        let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
+        let dataset = Dataset::open_ex(
+            format!("/vsizip/{}", fixture.path().display()),
+            dataset_options,
+        )
+        .unwrap();
         let root_group = dataset.root_group().unwrap();
-        let array_name = "Band1".to_string();
-        let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
+        let array_name = "byte_no_cf".to_string();
+        let options = CslStringList::new();
         let md_array = root_group.open_md_array(&array_name, options).unwrap();
 
         assert!(md_array.get_statistics(false, true).unwrap().is_none());