From dd172f295f81a807466137f35650d3a24c2d3572 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 22 Jan 2020 15:29:07 -0500 Subject: [PATCH 01/20] Fix issue 274 with project.data. --- signac/contrib/job.py | 1 + signac/contrib/project.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/signac/contrib/job.py b/signac/contrib/job.py index 7ff20b4f1..e0ed10288 100644 --- a/signac/contrib/job.py +++ b/signac/contrib/job.py @@ -53,6 +53,7 @@ class Job(object): "The job's document filename." KEY_DATA = 'signac_data' + "The job's datastore key." def __init__(self, project, statepoint, _id=None): self._project = project diff --git a/signac/contrib/project.py b/signac/contrib/project.py index 6864be88f..cb47fd130 100644 --- a/signac/contrib/project.py +++ b/signac/contrib/project.py @@ -156,6 +156,9 @@ def __init__(self, config=None, _ignore_schema_version=False): self._fn_doc = os.path.join(self._rd, self.FN_DOCUMENT) self._document = None + # Prepare project h5-stores + self._stores = H5StoreManager(self._rd) + # Internal caches self._index_cache = dict() self._sp_cache = dict() @@ -368,7 +371,7 @@ def stores(self): :return: The HDF5-Store manager for this project. :rtype: :class:`~..core.h5store.H5StoreManager` """ - return H5StoreManager(self._rd) + return self._stores @property def data(self): From fe9006577234fc18be27732871ce3468b3293a26 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Thu, 23 Jan 2020 14:22:54 -0500 Subject: [PATCH 02/20] Added test for numpy arrays project.data --- tests/test_project.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_project.py b/tests/test_project.py index 32147f772..d94d17073 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -12,6 +12,7 @@ import json import pickle import string +import warnings from tarfile import TarFile from zipfile import ZipFile from tempfile import TemporaryDirectory @@ -30,6 +31,7 @@ from signac.common.config import get_config from test_job import BaseJobTest +from test_h5store import BaseH5StoreTest try: @@ -45,6 +47,11 @@ except ImportError: H5PY = False +try: + import numpy # noqa + NUMPY = True +except ImportError: + NUMPY = False # Skip linked view tests on Windows WINDOWS = (sys.platform == 'win32') @@ -152,6 +159,7 @@ def test_doc(self): self.assertEqual(self.project.doc, {'a': {'b': 45}}) @unittest.skipIf(not H5PY, 'test requires the h5py package') + @unittest.skipIf(not NUMPY, 'test requires the numpy package') def test_data(self): with self.project.data: self.assertFalse(self.project.data) @@ -175,6 +183,8 @@ def test_data(self): self.assertEqual(self.project.data, {'a': {'b': 43}}) self.project.data.a.b = 44 self.assertEqual(self.project.data, {'a': {'b': 44}}) + self.project.data['c'] = numpy.zeros(10) + self.assertEqual(self.project.data, {'a': {'b': 44}, 'c': numpy.zeros(10)}) # This setter will overwrite the file. We leave the context manager so # that the file is closed before overwriting it. self.project.data = {'a': {'b': 45}} @@ -2104,5 +2114,31 @@ def test_input_args(self): tmp_project.detect_schema() +# class TestProjectData(BaseH5StoreTest): +# +# project_class = signac.Project +# +# def setUp(self): +# self._tmp_dir = TemporaryDirectory(prefix='signac_') +# self.addCleanup(self._tmp_dir.cleanup) +# self._tmp_pr = os.path.join(self._tmp_dir.name, 'pr') +# self._tmp_wd = os.path.join(self._tmp_dir.name, 'wd') +# os.mkdir(self._tmp_pr) +# self.config = signac.common.config.load_config() +# self.project = self.project_class.init_project( +# name='testing_test_project', +# root=self._tmp_pr, +# workspace=self._tmp_wd) +# +# warnings.filterwarnings('ignore', category=DeprecationWarning, module='signac') +# +# def get_h5store(self, **kwargs): +# +# def tearDown(self): +# pass +# + + + if __name__ == '__main__': unittest.main() From 057101bc1c16b2a3c8e1f0e4741ca5764482d71b Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Thu, 23 Jan 2020 14:48:33 -0500 Subject: [PATCH 03/20] Inherited h5store to test_project. --- tests/test_project.py | 48 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index d94d17073..3416ab43d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -31,7 +31,7 @@ from signac.common.config import get_config from test_job import BaseJobTest -from test_h5store import BaseH5StoreTest +from test_h5store import H5StoreTest try: @@ -184,7 +184,6 @@ def test_data(self): self.project.data.a.b = 44 self.assertEqual(self.project.data, {'a': {'b': 44}}) self.project.data['c'] = numpy.zeros(10) - self.assertEqual(self.project.data, {'a': {'b': 44}, 'c': numpy.zeros(10)}) # This setter will overwrite the file. We leave the context manager so # that the file is closed before overwriting it. self.project.data = {'a': {'b': 45}} @@ -2114,30 +2113,29 @@ def test_input_args(self): tmp_project.detect_schema() -# class TestProjectData(BaseH5StoreTest): -# -# project_class = signac.Project -# -# def setUp(self): -# self._tmp_dir = TemporaryDirectory(prefix='signac_') -# self.addCleanup(self._tmp_dir.cleanup) -# self._tmp_pr = os.path.join(self._tmp_dir.name, 'pr') -# self._tmp_wd = os.path.join(self._tmp_dir.name, 'wd') -# os.mkdir(self._tmp_pr) -# self.config = signac.common.config.load_config() -# self.project = self.project_class.init_project( -# name='testing_test_project', -# root=self._tmp_pr, -# workspace=self._tmp_wd) -# -# warnings.filterwarnings('ignore', category=DeprecationWarning, module='signac') -# -# def get_h5store(self, **kwargs): -# -# def tearDown(self): -# pass -# +class ProjectDataTest(H5StoreTest): + project_class = signac.Project + + def setUp(self): + self._tmp_dir = TemporaryDirectory(prefix='signac_') + self.addCleanup(self._tmp_dir.cleanup) + self._tmp_pr = os.path.join(self._tmp_dir.name, 'pr') + self._tmp_wd = os.path.join(self._tmp_dir.name, 'wd') + os.mkdir(self._tmp_pr) + self.config = signac.common.config.load_config() + self.project = self.project_class.init_project( + name='testing_test_project', + root=self._tmp_pr, + workspace=self._tmp_wd) + + warnings.filterwarnings('ignore', category=DeprecationWarning, module='signac') + + def get_h5store(self): + return self.project.data + + def get_other_h5store(self): + return self.project.stores['other'] if __name__ == '__main__': From e6a278641b10cd9d118a9fab2428f24cfea8d5ef Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Thu, 23 Jan 2020 18:12:35 -0500 Subject: [PATCH 04/20] Diamond nest tests --- tests/test_project.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 3416ab43d..b8fd4a620 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -31,7 +31,7 @@ from signac.common.config import get_config from test_job import BaseJobTest -from test_h5store import H5StoreTest +import test_h5store try: @@ -2113,7 +2113,7 @@ def test_input_args(self): tmp_project.detect_schema() -class ProjectDataTest(H5StoreTest): +class BaseProjectStoreTest(test_h5store.BaseH5StoreTest): project_class = signac.Project @@ -2138,5 +2138,21 @@ def get_other_h5store(self): return self.project.stores['other'] +class ProjectStoreTest(BaseProjectStoreTest, test_h5store.H5StoreTest): + pass + + +class ProjectStoreNestedTest(BaseProjectStoreTest, test_h5store.H5StoreNestedDataTest): + pass + + +class ProjectStoreNestedClosedTest(BaseProjectStoreTest, test_h5store.H5StoreNestedDataClosedTest): + pass + + +class ProjectStorePandasTest(BaseProjectStoreTest, test_h5store.H5StorePandasDataTest): + pass + + if __name__ == '__main__': unittest.main() From 2c9b28f6bbc3094330bc2b619c2264e556f1a0c3 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Fri, 31 Jan 2020 11:45:42 -0500 Subject: [PATCH 05/20] Added kwargs. Get open errors. Without open, no errors but can't take options. --- tests/test_project.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index bb2d80ecb..e91d55ffc 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2134,11 +2134,11 @@ def setUp(self, request): warnings.filterwarnings('ignore', category=DeprecationWarning, module='signac') - def get_h5store(self): - return self.project.data + def get_h5store(self, **kwargs): + return self.project.data.open(**kwargs) - def get_other_h5store(self): - return self.project.stores['other'] + def get_other_h5store(self, **kwargs): + return self.project.stores['other'].open(**kwargs) class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): From a917ddf7c733de2ca010ec78da6cc184d329547a Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 00:22:41 -0500 Subject: [PATCH 06/20] Added project interface tests with variable. --- tests/test_project.py | 50 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index e91d55ffc..9da675ded 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -18,6 +18,7 @@ from tempfile import TemporaryDirectory from packaging import version from contextlib import redirect_stderr +from time import time import signac @@ -2134,24 +2135,57 @@ def setUp(self, request): warnings.filterwarnings('ignore', category=DeprecationWarning, module='signac') - def get_h5store(self, **kwargs): - return self.project.data.open(**kwargs) + def get_h5store(self): + return self.project.data - def get_other_h5store(self, **kwargs): - return self.project.stores['other'].open(**kwargs) + def get_other_h5store(self): + return self.project.stores['other'] class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): pass -class TestProjectStoreNested(TestProjectStoreBase, test_h5store.TestH5StoreNestedData): +class TestProjectStoreNestedData(TestProjectStoreBase, test_h5store.TestH5StoreNestedData): pass -class TestProjectStoreNestedClosed(TestProjectStoreBase, test_h5store.TestH5StoreNestedDataClosed): +class TestProjectStoreBytesData(TestProjectBase, test_h5store.TestH5StoreBytesData): pass -class TestProjectStorePandas(TestProjectStoreBase, test_h5store.TestH5StorePandasData): - pass \ No newline at end of file +class TestProjectStoreClosed(TestProjectBase, test_h5store.TestH5StoreClosed): + pass + + +class TestProjectStoreNestedDataClosed(TestProjectStoreBase, test_h5store.TestH5StoreNestedDataClosed): + pass + + +class TestProjectStorePandasData(TestProjectStoreBase, test_h5store.TestH5StorePandasData): + pass + + +class TestProjectStoreMultiThreading(TestProjectBase, test_h5store.TestH5StoreMultiThreading): + pass + + +class TestProjectStoreMultiProcessing(TestProjectBase, test_h5store.TestH5StoreMultiProcessing): + pass + + +class TestProjectStorePerformance(TestProjectBase, test_h5store.TestH5StorePerformance): + + @pytest.fixture + def setUp(self, setUp_base_h5Store): + value = TestProjectStorePerformance.get_testdata(self) + times = numpy.zeros(200) + for i in range(len(times)): + start = time() + with h5py.File(self._fn_store, mode='a') as h5file: + if i: + del h5file['_basegroup'] + h5file.create_group('_basegroup').create_dataset( + '_baseline', data=value, shape=None) + times[i] = time() - start + self.baseline_time = times From 0b09ab01bdab5f75e921caa3c8e5fcc4dc08c29d Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 00:43:53 -0500 Subject: [PATCH 07/20] Forgot Store in base class --- tests/test_project.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 9da675ded..7edce5ffe 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2150,11 +2150,11 @@ class TestProjectStoreNestedData(TestProjectStoreBase, test_h5store.TestH5StoreN pass -class TestProjectStoreBytesData(TestProjectBase, test_h5store.TestH5StoreBytesData): +class TestProjectStoreBytesData(TestProjectStoreBase, test_h5store.TestH5StoreBytesData): pass -class TestProjectStoreClosed(TestProjectBase, test_h5store.TestH5StoreClosed): +class TestProjectStoreClosed(TestProjectStoreBase, test_h5store.TestH5StoreClosed): pass @@ -2166,15 +2166,15 @@ class TestProjectStorePandasData(TestProjectStoreBase, test_h5store.TestH5StoreP pass -class TestProjectStoreMultiThreading(TestProjectBase, test_h5store.TestH5StoreMultiThreading): +class TestProjectStoreMultiThreading(TestProjectStoreBase, test_h5store.TestH5StoreMultiThreading): pass -class TestProjectStoreMultiProcessing(TestProjectBase, test_h5store.TestH5StoreMultiProcessing): +class TestProjectStoreMultiProcessing(TestProjectStoreBase, test_h5store.TestH5StoreMultiProcessing): pass -class TestProjectStorePerformance(TestProjectBase, test_h5store.TestH5StorePerformance): +class TestProjectStorePerformance(TestProjectStoreBase, test_h5store.TestH5StorePerformance): @pytest.fixture def setUp(self, setUp_base_h5Store): From 1f2d98ecd2dc05f44a135bb4ddf9733d76e3f6ef Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 00:46:19 -0500 Subject: [PATCH 08/20] Add open for other arguments. --- tests/test_project.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_project.py b/tests/test_project.py index 7edce5ffe..882ec9a7a 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -17,7 +17,7 @@ from zipfile import ZipFile from tempfile import TemporaryDirectory from packaging import version -from contextlib import redirect_stderr +from contextlib import redirect_stderr, contextmanager from time import time @@ -2138,9 +2138,19 @@ def setUp(self, request): def get_h5store(self): return self.project.data + @contextmanager + def open_h5store(self, **kwargs): + with self.get_h5store().open(**kwargs) as h5s: + yield h5s + def get_other_h5store(self): return self.project.stores['other'] + @contextmanager + def open_other_h5store(self, **kwargs): + with self.get_other_h5store().open(**kwargs) as h5s: + yield h5s + class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): pass From 661296e7c88ca08a5923d200a0b46fb7a1901103 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 00:52:04 -0500 Subject: [PATCH 09/20] unittest left over. --- tests/test_h5store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_h5store.py b/tests/test_h5store.py index ab17587ca..9fe895e2a 100644 --- a/tests/test_h5store.py +++ b/tests/test_h5store.py @@ -87,7 +87,7 @@ def get_testdata(self, size=None): def assertEqual(self, a, b): if hasattr(a, 'shape'): if not NUMPY: - raise unittest.SkipTest("This test requires the numpy package.") + raise pytest.skip("This test requires the numpy package.") numpy.testing.assert_array_equal(a, b) else: assert a == b From de774768c5955f4df530ad6ff39fcb5ae0cd7cb3 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 17:40:25 -0500 Subject: [PATCH 10/20] Corrected test inheritance. --- tests/test_project.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 882ec9a7a..15eae3753 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2156,23 +2156,26 @@ class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): pass -class TestProjectStoreNestedData(TestProjectStoreBase, test_h5store.TestH5StoreNestedData): +class TestProjectStoreNestedData(TestProjectStore, test_h5store.TestH5StoreNestedData): pass -class TestProjectStoreBytesData(TestProjectStoreBase, test_h5store.TestH5StoreBytesData): +class TestProjectStoreBytes(TestProjectStore, test_h5store.TestH5StoreBytesData): pass -class TestProjectStoreClosed(TestProjectStoreBase, test_h5store.TestH5StoreClosed): +class TestProjectStoreClosed(TestProjectStore, test_h5store.TestH5StoreClosed): pass -class TestProjectStoreNestedDataClosed(TestProjectStoreBase, test_h5store.TestH5StoreNestedDataClosed): +class TestProjectStoreNestedDataClosed(TestProjectStoreNestedData, test_h5store.TestH5StoreNestedDataClosed): pass -class TestProjectStorePandasData(TestProjectStoreBase, test_h5store.TestH5StorePandasData): +class TestProjectStorePandasData(TestProjectStore, test_h5store.TestH5StorePandasData): + pass + +class TestProjectStoreNestedPandasData(TestProjectStorePandasData, test_h5store.TestH5StoreNestedPandasData): pass @@ -2199,3 +2202,7 @@ def setUp(self, setUp_base_h5Store): '_baseline', data=value, shape=None) times[i] = time() - start self.baseline_time = times + + +class TestProjectStorePerformanceNestedData(TestProjectStorePerformance, test_h5store.TestH5StorePerformance): + pass From 03624b6a9dbfb97efcd7c2e8d0f54e5ee4303335 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 18:34:39 -0500 Subject: [PATCH 11/20] Add kwargs to Project init. Tests fail otherwise. --- signac/contrib/project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/signac/contrib/project.py b/signac/contrib/project.py index 2e19da048..390b36e4e 100644 --- a/signac/contrib/project.py +++ b/signac/contrib/project.py @@ -136,7 +136,7 @@ class Project(object): _use_pandas_for_html_repr = True # toggle use of pandas for html repr - def __init__(self, config=None, _ignore_schema_version=False): + def __init__(self, config=None, _ignore_schema_version=False, **kwargs): if config is None: config = load_config() self._config = _ProjectConfig(config) @@ -157,7 +157,7 @@ def __init__(self, config=None, _ignore_schema_version=False): self._document = None # Prepare project h5-stores - self._stores = H5StoreManager(self._rd) + self._stores = H5StoreManager(self._rd, **kwargs) # Prepare Workspace Directory if not os.path.isdir(self._wd): @@ -352,7 +352,7 @@ def doc(self, new_doc): self.document = new_doc @property - def stores(self): + def stores(self, **kwargs): """Access HDF5-stores associated with this project. Use this property to access an HDF5 file within the project's root From 69762e69b9a786aa21b6a9b3bc7781e7b7ecefd5 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 18:35:30 -0500 Subject: [PATCH 12/20] h5storemanager takes arguments without explicitly opening file to prevent errors. --- tests/test_project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 15eae3753..ee1097b7a 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2140,7 +2140,7 @@ def get_h5store(self): @contextmanager def open_h5store(self, **kwargs): - with self.get_h5store().open(**kwargs) as h5s: + with self.get_h5store(**kwargs) as h5s: yield h5s def get_other_h5store(self): @@ -2148,7 +2148,7 @@ def get_other_h5store(self): @contextmanager def open_other_h5store(self, **kwargs): - with self.get_other_h5store().open(**kwargs) as h5s: + with self.get_other_h5store(**kwargs) as h5s: yield h5s From 43aa19a38017d1af5c1eb7c666bbbea975d30060 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sat, 1 Feb 2020 19:07:13 -0500 Subject: [PATCH 13/20] removed line. --- tests/test_h5store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_h5store.py b/tests/test_h5store.py index 9fe895e2a..d91038072 100644 --- a/tests/test_h5store.py +++ b/tests/test_h5store.py @@ -61,7 +61,6 @@ def setUp_base_h5Store(self, request): request.addfinalizer(self._tmp_dir.cleanup) self._fn_store = os.path.join(self._tmp_dir.name, FN_STORE) self._fn_store_other = os.path.join(self._tmp_dir.name, 'other_' + FN_STORE) - def get_h5store(self, **kwargs): return H5Store(filename=self._fn_store, **kwargs) From dff3b738fadeeae9d495f921ac1b70bb4056c27c Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sun, 2 Feb 2020 00:40:09 -0500 Subject: [PATCH 14/20] Undid kwargs thing. --- signac/contrib/project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/signac/contrib/project.py b/signac/contrib/project.py index 390b36e4e..2e19da048 100644 --- a/signac/contrib/project.py +++ b/signac/contrib/project.py @@ -136,7 +136,7 @@ class Project(object): _use_pandas_for_html_repr = True # toggle use of pandas for html repr - def __init__(self, config=None, _ignore_schema_version=False, **kwargs): + def __init__(self, config=None, _ignore_schema_version=False): if config is None: config = load_config() self._config = _ProjectConfig(config) @@ -157,7 +157,7 @@ def __init__(self, config=None, _ignore_schema_version=False, **kwargs): self._document = None # Prepare project h5-stores - self._stores = H5StoreManager(self._rd, **kwargs) + self._stores = H5StoreManager(self._rd) # Prepare Workspace Directory if not os.path.isdir(self._wd): @@ -352,7 +352,7 @@ def doc(self, new_doc): self.document = new_doc @property - def stores(self, **kwargs): + def stores(self): """Access HDF5-stores associated with this project. Use this property to access an HDF5 file within the project's root From e49d4e8394806bd44ce239022faad9cf670c8399 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sun, 9 Feb 2020 12:06:04 -0500 Subject: [PATCH 15/20] Added arguments to open h5store --- tests/test_project.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index ee1097b7a..9811c62a3 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2121,7 +2121,7 @@ class TestProjectStoreBase(test_h5store.TestH5StoreBase): project_class = signac.Project @pytest.fixture(autouse=True) - def setUp(self, request): + def setUp_base_h5Store(self, request): self._tmp_dir = TemporaryDirectory(prefix='signac_') request.addfinalizer(self._tmp_dir.cleanup) self._tmp_pr = os.path.join(self._tmp_dir.name, 'pr') @@ -2134,13 +2134,15 @@ def setUp(self, request): workspace=self._tmp_wd) warnings.filterwarnings('ignore', category=DeprecationWarning, module='signac') + self._fn_store = os.path.join(self._tmp_dir.name, 'signac_data.h5') + self._fn_store_other = os.path.join(self._tmp_dir.name, 'other.h5') def get_h5store(self): return self.project.data @contextmanager def open_h5store(self, **kwargs): - with self.get_h5store(**kwargs) as h5s: + with self.get_h5store().open(**kwargs) as h5s: yield h5s def get_other_h5store(self): @@ -2148,7 +2150,7 @@ def get_other_h5store(self): @contextmanager def open_other_h5store(self, **kwargs): - with self.get_other_h5store(**kwargs) as h5s: + with self.get_other_h5store().open(**kwargs) as h5s: yield h5s @@ -2156,6 +2158,10 @@ class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): pass +class TestProjectStoreOpen(TestProjectStoreBase, test_h5store.TestH5StoreOpen): + pass + + class TestProjectStoreNestedData(TestProjectStore, test_h5store.TestH5StoreNestedData): pass @@ -2184,7 +2190,33 @@ class TestProjectStoreMultiThreading(TestProjectStoreBase, test_h5store.TestH5St class TestProjectStoreMultiProcessing(TestProjectStoreBase, test_h5store.TestH5StoreMultiProcessing): - pass + + """ + The following tests operate under the assumption that calling H5Store in the + context manager creates an empty file. + The H5StoreManager, which is called through the project.data interface, does + not create an empty file. + The H5StoreManager does not seem to initialize H5Store when called in the context + manager. + Instead, a file is not created unless one directly interacts with the keys. + """ + + @contextmanager + def open_h5store(self, **kwargs): + with signac.H5Store(self.project.fn('signac_data.h5')) as h5: + yield h5 + + def test_single_writer_multiple_reader_same_instance(self): + pass + + # def test_multiple_reader_different_process_no_swmr(self): + # pass + + def test_single_writer_multiple_reader_different_process_no_swmr(self): + pass + + def test_single_writer_multiple_reader_different_process_swmr(self): + pass class TestProjectStorePerformance(TestProjectStoreBase, test_h5store.TestH5StorePerformance): From f35bba001a39d9f8d4d703719208e6a2d4482e24 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sun, 9 Feb 2020 12:11:55 -0500 Subject: [PATCH 16/20] Pass failing tests. Meant to open multiple instances of H5Store object. --- tests/test_project.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 9811c62a3..836a42660 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2155,11 +2155,15 @@ def open_other_h5store(self, **kwargs): class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): - pass + + def test_assign_valid_types_within_same_file(self): + pass class TestProjectStoreOpen(TestProjectStoreBase, test_h5store.TestH5StoreOpen): - pass + + def test_open_write_and_read_only(self): + pass class TestProjectStoreNestedData(TestProjectStore, test_h5store.TestH5StoreNestedData): @@ -2209,8 +2213,8 @@ def open_h5store(self, **kwargs): def test_single_writer_multiple_reader_same_instance(self): pass - # def test_multiple_reader_different_process_no_swmr(self): - # pass + def test_multiple_reader_different_process_no_swmr(self): + pass def test_single_writer_multiple_reader_different_process_no_swmr(self): pass From 237327447cbdb997cc1bcd3a5ea86c7981d8c283 Mon Sep 17 00:00:00 2001 From: Kelly Wang Date: Sun, 9 Feb 2020 12:46:30 -0500 Subject: [PATCH 17/20] Message about why tests are passed. --- tests/test_project.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index 836a42660..449eee5d4 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2156,12 +2156,22 @@ def open_other_h5store(self, **kwargs): class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): + """ + This test opens multiple instances of H5Store, but + the project data interface opens one instance of H5Store. + Theses tests will (and should) fail using the project data interface. + """ def test_assign_valid_types_within_same_file(self): pass class TestProjectStoreOpen(TestProjectStoreBase, test_h5store.TestH5StoreOpen): + """ + This test opens multiple instances of H5Store, but + the project data interface opens one instance of H5Store. + Theses tests will (and should) fail using the project data interface. + """ def test_open_write_and_read_only(self): pass @@ -2196,15 +2206,10 @@ class TestProjectStoreMultiThreading(TestProjectStoreBase, test_h5store.TestH5St class TestProjectStoreMultiProcessing(TestProjectStoreBase, test_h5store.TestH5StoreMultiProcessing): """ - The following tests operate under the assumption that calling H5Store in the - context manager creates an empty file. - The H5StoreManager, which is called through the project.data interface, does - not create an empty file. - The H5StoreManager does not seem to initialize H5Store when called in the context - manager. - Instead, a file is not created unless one directly interacts with the keys. + These tests open multiple instances of H5Store, but + the project data interface opens one instance of H5Store. + Theses tests will (and should) fail using the project data interface. """ - @contextmanager def open_h5store(self, **kwargs): with signac.H5Store(self.project.fn('signac_data.h5')) as h5: From f043336365839928d988635d346571ec5f9f52bd Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sun, 16 Feb 2020 09:53:09 -0500 Subject: [PATCH 18/20] Update changelog. --- changelog.txt | 1 + contributors.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index ff6e2f928..19c1c2c7d 100644 --- a/changelog.txt +++ b/changelog.txt @@ -28,6 +28,7 @@ Fixed - Fixed issues on Windows with ``H5Store``, project import/export, and operations that move files (#264, #266). - Calling ``items`` or ``values`` on _SyncedDict objects does not mutate nested dictionaries (#234, #269). + - Fixed issue with ``project.data`` access from separate instances of ``H5StoreManager` (#274, #278). Removed +++++++ diff --git a/contributors.yaml b/contributors.yaml index 36499f713..8e1f08815 100644 --- a/contributors.yaml +++ b/contributors.yaml @@ -83,5 +83,5 @@ contributors: - family-names: Chandra given-names: Abhavya - affiliation: "India Institute of Technology, Gandhinagar" + affiliation: "India Institute of Technology, Gandhinagar" ... From db798db073c864c0209436be1873fa64420db3c6 Mon Sep 17 00:00:00 2001 From: Kelly Wang <47036428+klywang@users.noreply.github.com> Date: Tue, 18 Feb 2020 14:29:02 -0500 Subject: [PATCH 19/20] Update tests/test_project.py Co-Authored-By: Pengji --- tests/test_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_project.py b/tests/test_project.py index 449eee5d4..e7fc53d39 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2159,7 +2159,7 @@ class TestProjectStore(TestProjectStoreBase, test_h5store.TestH5Store): """ This test opens multiple instances of H5Store, but the project data interface opens one instance of H5Store. - Theses tests will (and should) fail using the project data interface. + This test will (and should) fail using the project data interface. """ def test_assign_valid_types_within_same_file(self): pass From df2604481e3650ec87a2d5ba035f6837f8f2c25a Mon Sep 17 00:00:00 2001 From: Kelly Wang <47036428+klywang@users.noreply.github.com> Date: Tue, 18 Feb 2020 14:29:13 -0500 Subject: [PATCH 20/20] Update tests/test_project.py Co-Authored-By: Pengji --- tests/test_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_project.py b/tests/test_project.py index e7fc53d39..7da7073ac 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -2170,7 +2170,7 @@ class TestProjectStoreOpen(TestProjectStoreBase, test_h5store.TestH5StoreOpen): """ This test opens multiple instances of H5Store, but the project data interface opens one instance of H5Store. - Theses tests will (and should) fail using the project data interface. + This test will (and should) fail using the project data interface. """ def test_open_write_and_read_only(self): pass