Merge pull request #89 from martindurant/cdf_attrs

Remove CDF special attributes when translating
fsspec · Oct 15, 2021 · b9110d0 · b9110d0
2 parents 6cbe2e6 + cf2a1ee
commit b9110d0
Show file tree

Hide file tree

Showing 7 changed files with 68 additions and 90 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/fsspec_reference_maker/hdf.py b/fsspec_reference_maker/hdf.py
@@ -1,19 +1,23 @@
 import base64
-import zipfile
 from typing import Union, BinaryIO
 import logging
-import os
-import ujson as json
 import numpy as np
 import h5py
 import zarr
 from zarr.meta import encode_fill_value
 import numcodecs
-import fsspec
-import fsspec.utils
-import fsspec.core
 
 lggr = logging.getLogger('h5-to-zarr')
+_HIDDEN_ATTRS = {  # from h5netcdf.attrs
+    "REFERENCE_LIST",
+    "CLASS",
+    "DIMENSION_LIST",
+    "NAME",
+    "_Netcdf4Dimid",
+    "_Netcdf4Coordinates",
+    "_nc3_strict",
+    "_NCProperties",
+}
 
 
 class SingleHdf5ToZarr:
@@ -37,7 +41,7 @@ class SingleHdf5ToZarr:
     """
 
     def __init__(self, h5f: BinaryIO, url: str,
-                 spec=1, inline_threshold=0):
+                 spec=1, inline_threshold=100):
         # Open HDF5 file in read mode...
         lggr.debug(f'HDF5 file: {h5f}')
         self.input_file = h5f
@@ -115,7 +119,7 @@ def _transfer_attrs(self, h5obj: Union[h5py.Dataset, h5py.Group],
             attributes.
         """
         for n, v in h5obj.attrs.items():
-            if n in ('REFERENCE_LIST', 'DIMENSION_LIST'):
+            if n in _HIDDEN_ATTRS:
                 continue
 
             # Fix some attribute values to avoid JSON encoding exceptions...

diff --git a/tests/example_jsons/multizarr_example.json b/tests/example_jsons/multizarr_example.json
diff --git a/tests/example_jsons/multizarr_notemplates_example.json b/tests/example_jsons/multizarr_notemplates_example.json
diff --git a/tests/example_jsons/single_example.json b/tests/example_jsons/single_example.json
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -12,3 +12,4 @@ fastparquet
 s3fs>=2021.08.0
 jinja2
 dask
+h5netcdf
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,3 +12,4 @@ fastparquet @@
     s3fs>=2021.08.0
     jinja2
     dask
+    h5netcdf