Merge pull request #15 from cedadev/edit_view

2024.11.6

README.md

@@ -12,7 +12,6 @@ For use with the Xarray module as an additional backend.
 # Installation
 
 ```
-pip install xarray==2024.6.0
 pip install cfapyx
 ```
 

cfapyx/creator.py

@@ -428,7 +428,7 @@ def _determine_size_opts(self, var_info: dict, agg_dims: list) -> list:
             cds = v['dims']
 
             if (set(agg_dims) & set(cds)):
-                if cds and sorted(cds) not in cdimopts:
+                if cds and cds not in cdimopts:
                     cdimopts.append(cds)
 
         logger.debug(f'Determined {len(cdimopts)} size options:')
@@ -460,6 +460,10 @@ def _accumulate_attrs(self, attrs: dict, ncattrs: dict) -> dict:
                     logger.warning(f'AttributeWarning: Attribute "{attr}" not present in all files')
                     attrs[attr] = self.concat_msg
             else:
+                if isinstance(ncattrs[attr], (np.ndarray, np.generic)):
+                    if not np.array_equal(attrs[attr], ncattrs[attr]):
+                        attrs[attr] = self.concat_msg
+                    continue
                 if attrs[attr] != ncattrs[attr]:
                     attrs[attr] = self.concat_msg
                 else:
@@ -849,6 +853,69 @@ def write(
 
         self.ds.close()
 
+    def display_attrs(self):
+        """
+        Display the global attributes consolidated in the 
+        aggregation process.
+        """
+        print('Global Attributes:')
+        _display_attrs(self.global_attrs)
+
+    def display_variables(self):
+        """
+        Display the variables and some basic properties about each.
+        """
+        if not self.var_info:
+            return
+
+        for v, m in self.var_info.items():
+            print(f'{v}: dtype={m["dtype"]}, dimensions={m["dims"]}')
+
+    def display_dimensions(self):
+        """
+        Display the dimensions and some basic properties about each.
+        """
+        if not self.dim_info:
+            return
+
+        for d, m in self.dim_info.items():
+            print(f'{d}: dtype={m["dtype"]}, size={m["size"]}', end='')
+            if 'f_size' in m:
+                print(f', f_size={m["f_size"]}')
+            else:
+                print()
+
+    def display_variable(self, var):
+        """
+        Handler for displaying information about a variable
+        """
+        if not self.var_info:
+            return ''
+        if var not in self.var_info:
+            return ''
+        self._display_item(self.var_info[var])
+
+    def display_dimension(self, dim):
+        """
+        Handler for displaying information about a variable
+        """
+        if not self.dim_info:
+            return ''
+        if dim not in self.dim_info:
+            return ''
+        self._display_item(self.dim_info[dim])
+
+    def _display_item(self, keyset):
+        """
+        Display the information about a dimension/variable
+        """
+        for attr, value in keyset.items():
+            if attr == 'attrs':
+                print('Attributes:')
+                _display_attrs(value)
+            else:
+                print(f'{attr}: {value}')
+
     @property
     def agg_dims(self):
         """
@@ -1064,3 +1131,7 @@ def _call_file(self, file: str) -> netCDF4.Dataset:
                 self.longest_filename = file
 
         return ds
+
+def _display_attrs(attrs):
+    for k, v in attrs.items():
+        print(f' - {k}: {v}')

cfapyx/tests/conftest.py

@@ -0,0 +1,23 @@
+
+def pytest_collection_modifyitems(items):
+
+    CLASS_ORDER = [
+        "TestPath"
+        "TestCFAWrite",
+        "TestCFARead",
+        "TestCleanup"
+    ]
+
+    sorted_items = items.copy()
+      # read the class names from default items
+    class_mapping = {item: item.cls.__name__ for item in items}
+
+
+    # Iteratively move tests of each class to the end of the test queue
+    for class_ in CLASS_ORDER:
+        sorted_items = [it for it in sorted_items if class_mapping[it] != class_] + [
+            it for it in sorted_items if class_mapping[it] == class_
+        ]
+
+
+    items[:] = sorted_items

cfapyx/tests/test_basic.py

@@ -0,0 +1,15 @@
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+TESTDIR = 'cfapyx/tests/test_space'
+
+class TestPath:
+    def testpath(self, tests=TESTDIR):
+        logger.error(os.getcwd())
+        logger.error(tests)
+
+class TestCleanup:
+    def test_cleanup(self, testdir=TESTDIR):
+        os.system(f'rm {testdir}/testrain.nca')
+        print('Integration tests: Cleanup - complete')

cfapyx/tests/test_read.py

@@ -0,0 +1,90 @@
+# All routines for testing CFA general methods.
+import xarray as xr
+import os
+import pytest
+
+TESTDIR = 'cfapyx/tests/test_space'
+
+class TestCFARead:
+
+    def test_cfa_pure(self, testdir=TESTDIR, active=False):
+
+        FILE = f'{testdir}/testrain.nca'
+
+        # Local testing: Add CFAPyX before tests
+        try:
+            ds = xr.open_dataset(FILE, engine='CFA',
+                                cfa_options={
+                                    'use_active':active
+                                    })
+        except Exception as err:
+            assert isinstance(err, ImportError)
+            print(f'Integration tests: Read(pure, active={active}) - skipped')
+            return
+
+        ## Test global dataset
+        assert not hasattr(ds,'address')
+        assert not hasattr(ds,'shape')
+        assert not hasattr(ds,'location')
+
+        assert 'p' in ds
+        assert ds['p'].shape == (20, 180, 360)
+
+        # Using selection with lat/lon values NOT index values
+        p_sel = ds['p'].isel(time=slice(0,3),latitude=slice(140,145), longitude=slice(90,100))
+
+        assert p_sel.shape == (3, 5, 10)
+        assert not hasattr(p_sel, 'aggregated_data')
+        assert not hasattr(p_sel, 'aggregated_dimensions')
+
+        p_mean = p_sel.mean(dim='time')
+
+        assert p_mean.shape == (5, 10)
+        assert (p_mean[0][0].to_numpy() - 0.635366) < 0.01
+
+        p_value = p_sel.mean()
+
+        assert p_value.shape == ()
+        assert (p_value.to_numpy() - 0.511954) < 0.01
+
+        print(f'Integration tests: Read(pure, active={active}) - complete')
+
+    def test_cfa_chunks(self, testdir=TESTDIR):
+
+        FILE = f'{testdir}/testrain.nca'
+
+        ds = xr.open_dataset(FILE, engine='CFA',
+                            cfa_options={
+                                'chunks': {'longitude':180},
+                                'chunk_limits':False})
+
+        ## Test global dataset
+        assert not hasattr(ds,'address')
+        assert not hasattr(ds,'shape')
+        assert not hasattr(ds,'location')
+
+        assert 'p' in ds
+        assert ds['p'].shape == (20, 180, 360)
+
+        # Using selection with lat/lon values NOT index values
+        p_sel = ds['p'].isel(time=slice(0,3),latitude=slice(140,145), longitude=slice(100,300))
+
+        assert p_sel.shape == (3, 5, 200)
+        assert not hasattr(p_sel, 'aggregated_data')
+        assert not hasattr(p_sel, 'aggregated_dimensions')
+
+        p_mean = p_sel.mean(dim='time')
+
+        assert p_mean.shape == (5, 200)
+        assert (p_mean[0][0].to_numpy() - 0.664414) < 0.01
+
+        p_value = p_sel.mean()
+
+        assert p_value.shape == ()
+        assert (p_value.to_numpy() - 0.490389) < 0.01
+
+        print(f'Integration tests: Read(chunked) - complete')
+
+if __name__ == '__main__':
+
+    print('Run with poetry run pytest -v -s')

cfapyx/tests/test_write.py

@@ -0,0 +1,31 @@
+import os
+
+from cfapyx import CFANetCDF
+
+TESTDIR = 'cfapyx/tests/test_space'
+
+class TestCFAWrite:
+    def test_cfa_write(self, testdir=TESTDIR):
+
+
+        os.system(f'touch {testdir.replace("/","_")}')
+
+        filepattern = f'{testdir}/rain/example*.nc'
+
+        ds = CFANetCDF(filepattern, concat_msg='ThisIsATests')
+
+        ds.create(
+            updates={'test_name':'alpha', 'test_remove':'not removed'},
+            removals={'test_remove'}
+        )
+
+        assert ds.agg_dims == ('time',)
+        assert ds.coord_dims == ('time', 'latitude', 'longitude')
+        assert ds.pure_dims == ()
+        assert ds.aggregated_vars == ('p',)
+        assert ds.identical_vars == ()
+        assert ds.scalar_vars == ()
+
+        ds.write(f'{testdir}/testrain.nca')
+
+        print('Integration tests: Write - complete')

cfapyx/wrappers.py

@@ -336,7 +336,7 @@ def global_combine(internal, external):
                     cumul = max(filter(lambda l: l <= c, cumulative))
 
                 fc = np.where(cumulative == cumul)[0]
-                fragment_coord.append(int(fc))
+                fragment_coord.append(int(fc.squeeze()))
 
                 ext = slice(
                     partition_cumul[dim][c],