From 77e9cc91856ef9662a8844b26f7397ba0ada05fe Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 14:40:17 -0600 Subject: [PATCH 01/10] cp to tests --- tests/test_autopopulate.py | 158 +++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 tests/test_autopopulate.py diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py new file mode 100644 index 00000000..7a0a58e3 --- /dev/null +++ b/tests/test_autopopulate.py @@ -0,0 +1,158 @@ +from nose.tools import assert_equal, assert_false, assert_true, raises +from . import schema, PREFIX +from datajoint import DataJointError +import datajoint as dj + + +class TestPopulate: + """ + Test base relations: insert, delete + """ + + def setUp(self): + self.user = schema.User() + self.subject = schema.Subject() + self.experiment = schema.Experiment() + self.trial = schema.Trial() + self.ephys = schema.Ephys() + self.channel = schema.Ephys.Channel() + + def tearDown(self): + # delete automatic tables just in case + self.channel.delete_quick() + self.ephys.delete_quick() + self.trial.Condition.delete_quick() + self.trial.delete_quick() + self.experiment.delete_quick() + + def test_populate(self): + # test simple populate + assert_true(self.subject, "root tables are empty") + assert_false(self.experiment, "table already filled?") + self.experiment.populate() + assert_true( + len(self.experiment) + == len(self.subject) * self.experiment.fake_experiments_per_subject + ) + + # test restricted populate + assert_false(self.trial, "table already filled?") + restriction = self.subject.proj(animal="subject_id").fetch("KEY")[0] + d = self.trial.connection.dependencies + d.load() + self.trial.populate(restriction) + assert_true(self.trial, "table was not populated") + key_source = self.trial.key_source + assert_equal(len(key_source & self.trial), len(key_source & restriction)) + assert_equal(len(key_source - self.trial), len(key_source - restriction)) + + # test subtable populate + assert_false(self.ephys) + assert_false(self.channel) + self.ephys.populate() + assert_true(self.ephys) + assert_true(self.channel) + + def test_populate_with_success_count(self): + # test simple populate + assert_true(self.subject, "root tables are empty") + assert_false(self.experiment, "table already filled?") + ret = self.experiment.populate() + success_count = ret["success_count"] + assert_equal(len(self.experiment.key_source & self.experiment), success_count) + + # test restricted populate + assert_false(self.trial, "table already filled?") + restriction = self.subject.proj(animal="subject_id").fetch("KEY")[0] + d = self.trial.connection.dependencies + d.load() + ret = self.trial.populate(restriction, suppress_errors=True) + success_count = ret["success_count"] + assert_equal(len(self.trial.key_source & self.trial), success_count) + + def test_populate_exclude_error_and_ignore_jobs(self): + # test simple populate + assert_true(self.subject, "root tables are empty") + assert_false(self.experiment, "table already filled?") + + keys = self.experiment.key_source.fetch("KEY", limit=2) + for idx, key in enumerate(keys): + if idx == 0: + schema.schema.jobs.ignore(self.experiment.table_name, key) + else: + schema.schema.jobs.error(self.experiment.table_name, key, "") + + self.experiment.populate(reserve_jobs=True) + assert_equal( + len(self.experiment.key_source & self.experiment), + len(self.experiment.key_source) - 2, + ) + + def test_allow_direct_insert(self): + assert_true(self.subject, "root tables are empty") + key = self.subject.fetch("KEY", limit=1)[0] + key["experiment_id"] = 1000 + key["experiment_date"] = "2018-10-30" + self.experiment.insert1(key, allow_direct_insert=True) + + def test_multi_processing(self): + assert self.subject, "root tables are empty" + assert not self.experiment, "table already filled?" + self.experiment.populate(processes=2) + assert ( + len(self.experiment) + == len(self.subject) * self.experiment.fake_experiments_per_subject + ) + + def test_max_multi_processing(self): + assert self.subject, "root tables are empty" + assert not self.experiment, "table already filled?" + self.experiment.populate(processes=None) + assert ( + len(self.experiment) + == len(self.subject) * self.experiment.fake_experiments_per_subject + ) + + @raises(DataJointError) + def test_allow_insert(self): + assert_true(self.subject, "root tables are empty") + key = self.subject.fetch("KEY")[0] + key["experiment_id"] = 1001 + key["experiment_date"] = "2018-10-30" + self.experiment.insert1(key) + + def test_load_dependencies(self): + schema = dj.Schema(f"{PREFIX}_load_dependencies_populate") + + @schema + class ImageSource(dj.Lookup): + definition = """ + image_source_id: int + """ + contents = [(0,)] + + @schema + class Image(dj.Imported): + definition = """ + -> ImageSource + --- + image_data: longblob + """ + + def make(self, key): + self.insert1(dict(key, image_data=dict())) + + Image.populate() + + @schema + class Crop(dj.Computed): + definition = """ + -> Image + --- + crop_image: longblob + """ + + def make(self, key): + self.insert1(dict(key, crop_image=dict())) + + Crop.populate() From 22cfc2228ec3d20f6e65d62c5b44e92edc1ccbfd Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 14:55:11 -0600 Subject: [PATCH 02/10] First pass at migrating test_autopopulate --- tests/test_autopopulate.py | 87 ++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index 7a0a58e3..cb035fa2 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -1,79 +1,77 @@ -from nose.tools import assert_equal, assert_false, assert_true, raises +import pytest from . import schema, PREFIX from datajoint import DataJointError import datajoint as dj +@pytest.fixture +def schema_any_with_teardown(schema_any): + yield schema_any + # delete automatic tables just in case + schema_any.Ephys.Channel().delete_quick() + schema_any.Ephys().delete_quick() + schema_any.Trial().Condition.delete_quick() + schema_any.Trial().delete_quick() + schema_any.Experiment().delete_quick() + + class TestPopulate: """ Test base relations: insert, delete """ - def setUp(self): - self.user = schema.User() - self.subject = schema.Subject() - self.experiment = schema.Experiment() - self.trial = schema.Trial() - self.ephys = schema.Ephys() - self.channel = schema.Ephys.Channel() - - def tearDown(self): - # delete automatic tables just in case - self.channel.delete_quick() - self.ephys.delete_quick() - self.trial.Condition.delete_quick() - self.trial.delete_quick() - self.experiment.delete_quick() - - def test_populate(self): + def test_populate(self, schema_any_with_teardown): + breakpoint() # test simple populate - assert_true(self.subject, "root tables are empty") - assert_false(self.experiment, "table already filled?") + assert self.subject, "root tables are empty" + assert not self.experiment, "table already filled?" self.experiment.populate() - assert_true( + assert ( len(self.experiment) == len(self.subject) * self.experiment.fake_experiments_per_subject ) # test restricted populate - assert_false(self.trial, "table already filled?") + assert not self.trial, "table already filled?" restriction = self.subject.proj(animal="subject_id").fetch("KEY")[0] d = self.trial.connection.dependencies d.load() self.trial.populate(restriction) - assert_true(self.trial, "table was not populated") + assert self.trial, "table was not populated" key_source = self.trial.key_source - assert_equal(len(key_source & self.trial), len(key_source & restriction)) - assert_equal(len(key_source - self.trial), len(key_source - restriction)) + assert len(key_source & self.trial) == len(key_source & restriction) + assert len(key_source - self.trial) == len(key_source - restriction) # test subtable populate - assert_false(self.ephys) - assert_false(self.channel) + assert not self.ephys + assert not self.channel self.ephys.populate() - assert_true(self.ephys) - assert_true(self.channel) + assert self.ephys + assert self.channel + @pytest.mark.skip(reason="temp") def test_populate_with_success_count(self): # test simple populate - assert_true(self.subject, "root tables are empty") - assert_false(self.experiment, "table already filled?") + assert self.subject, "root tables are empty" + assert not self.experiment, "table already filled?" ret = self.experiment.populate() success_count = ret["success_count"] - assert_equal(len(self.experiment.key_source & self.experiment), success_count) + assert len(self.experiment.key_source & self.experiment) == success_count # test restricted populate - assert_false(self.trial, "table already filled?") + assert not self.trial, "table already filled?" restriction = self.subject.proj(animal="subject_id").fetch("KEY")[0] d = self.trial.connection.dependencies d.load() ret = self.trial.populate(restriction, suppress_errors=True) success_count = ret["success_count"] - assert_equal(len(self.trial.key_source & self.trial), success_count) + assert len(self.trial.key_source & self.trial) == success_count + @pytest.mark.skip(reason="temp") def test_populate_exclude_error_and_ignore_jobs(self): # test simple populate - assert_true(self.subject, "root tables are empty") - assert_false(self.experiment, "table already filled?") + assert self.subject, "root tables are empty" + assert not self.experiment, "table already filled?" keys = self.experiment.key_source.fetch("KEY", limit=2) for idx, key in enumerate(keys): @@ -83,18 +81,20 @@ def test_populate_exclude_error_and_ignore_jobs(self): schema.schema.jobs.error(self.experiment.table_name, key, "") self.experiment.populate(reserve_jobs=True) - assert_equal( - len(self.experiment.key_source & self.experiment), + assert ( + len(self.experiment.key_source & self.experiment) == len(self.experiment.key_source) - 2, ) + @pytest.mark.skip(reason="temp") def test_allow_direct_insert(self): - assert_true(self.subject, "root tables are empty") + assert self.subject, "root tables are empty" key = self.subject.fetch("KEY", limit=1)[0] key["experiment_id"] = 1000 key["experiment_date"] = "2018-10-30" self.experiment.insert1(key, allow_direct_insert=True) + @pytest.mark.skip(reason="temp") def test_multi_processing(self): assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" @@ -104,6 +104,7 @@ def test_multi_processing(self): == len(self.subject) * self.experiment.fake_experiments_per_subject ) + @pytest.mark.skip(reason="temp") def test_max_multi_processing(self): assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" @@ -113,14 +114,16 @@ def test_max_multi_processing(self): == len(self.subject) * self.experiment.fake_experiments_per_subject ) - @raises(DataJointError) + @pytest.mark.skip(reason="temp") def test_allow_insert(self): - assert_true(self.subject, "root tables are empty") + assert self.subject, "root tables are empty" key = self.subject.fetch("KEY")[0] key["experiment_id"] = 1001 key["experiment_date"] = "2018-10-30" - self.experiment.insert1(key) + with pytest.raises(DataJointError): + self.experiment.insert1(key) + @pytest.mark.skip(reason="temp") def test_load_dependencies(self): schema = dj.Schema(f"{PREFIX}_load_dependencies_populate") From f067017391223bc9786717fff4a53ba43b9fbb49 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 20:55:33 +0000 Subject: [PATCH 03/10] Format with black --- tests/test_autopopulate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index cb035fa2..ef47cc1b 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -82,8 +82,8 @@ def test_populate_exclude_error_and_ignore_jobs(self): self.experiment.populate(reserve_jobs=True) assert ( - len(self.experiment.key_source & self.experiment) == - len(self.experiment.key_source) - 2, + len(self.experiment.key_source & self.experiment) + == len(self.experiment.key_source) - 2, ) @pytest.mark.skip(reason="temp") From 78e1ed2818b0ee713bf86fa8412d342edd523028 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 15:11:04 -0600 Subject: [PATCH 04/10] Use setup and teardown class methods instead --- tests/test_autopopulate.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index ef47cc1b..5802e7f9 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -4,24 +4,30 @@ import datajoint as dj -@pytest.fixture -def schema_any_with_teardown(schema_any): - yield schema_any - # delete automatic tables just in case - schema_any.Ephys.Channel().delete_quick() - schema_any.Ephys().delete_quick() - schema_any.Trial().Condition.delete_quick() - schema_any.Trial().delete_quick() - schema_any.Experiment().delete_quick() - - class TestPopulate: """ Test base relations: insert, delete """ - def test_populate(self, schema_any_with_teardown): - breakpoint() + @classmethod + def setup_class(cls): + cls.user = schema.User() + cls.subject = schema.Subject() + cls.experiment = schema.Experiment() + cls.trial = schema.Trial() + cls.ephys = schema.Ephys() + cls.channel = schema.Ephys.Channel() + + @classmethod + def teardown_class(cls): + # Delete automatic tables just in case + cls.channel.delete_quick() + cls.ephys.delete_quick() + cls.trial.Condition.delete_quick() + cls.trial.delete_quick() + cls.experiment.delete_quick() + + def test_populate(self, schema_any): # test simple populate assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" From 4f2a3ca3b81ea9109dc18af0926e6040d5e4bf37 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 15:11:34 -0600 Subject: [PATCH 05/10] Teardown tolerates nonexistent table --- tests/test_autopopulate.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index 5802e7f9..282dc113 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -2,6 +2,7 @@ from . import schema, PREFIX from datajoint import DataJointError import datajoint as dj +import pymysql class TestPopulate: @@ -20,12 +21,15 @@ def setup_class(cls): @classmethod def teardown_class(cls): - # Delete automatic tables just in case - cls.channel.delete_quick() - cls.ephys.delete_quick() - cls.trial.Condition.delete_quick() - cls.trial.delete_quick() - cls.experiment.delete_quick() + """Delete automatic tables just in case""" + for autopop_table in ( + cls.channel, cls.ephys, cls.trial.Condition, cls.trial, cls.experiment + ): + try: + autopop_table.delete_quick() + except pymysql.err.OperationalError: + # Table doesn't exist + pass def test_populate(self, schema_any): # test simple populate From ebc2144b06049ab6bae1bf330603cda1730d43e9 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 15:18:32 -0600 Subject: [PATCH 06/10] Migrate test_autopopulate.py --- tests/test_autopopulate.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index 282dc113..d72cbc2c 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -59,8 +59,7 @@ def test_populate(self, schema_any): assert self.ephys assert self.channel - @pytest.mark.skip(reason="temp") - def test_populate_with_success_count(self): + def test_populate_with_success_count(self, schema_any): # test simple populate assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" @@ -77,8 +76,7 @@ def test_populate_with_success_count(self): success_count = ret["success_count"] assert len(self.trial.key_source & self.trial) == success_count - @pytest.mark.skip(reason="temp") - def test_populate_exclude_error_and_ignore_jobs(self): + def test_populate_exclude_error_and_ignore_jobs(self, schema_any): # test simple populate assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" @@ -86,26 +84,21 @@ def test_populate_exclude_error_and_ignore_jobs(self): keys = self.experiment.key_source.fetch("KEY", limit=2) for idx, key in enumerate(keys): if idx == 0: - schema.schema.jobs.ignore(self.experiment.table_name, key) + schema_any.jobs.ignore(self.experiment.table_name, key) else: - schema.schema.jobs.error(self.experiment.table_name, key, "") + schema_any.jobs.error(self.experiment.table_name, key, "") self.experiment.populate(reserve_jobs=True) - assert ( - len(self.experiment.key_source & self.experiment) - == len(self.experiment.key_source) - 2, - ) + assert len(self.experiment.key_source & self.experiment) == len(self.experiment.key_source) - 2 - @pytest.mark.skip(reason="temp") - def test_allow_direct_insert(self): + def test_allow_direct_insert(self, schema_any): assert self.subject, "root tables are empty" key = self.subject.fetch("KEY", limit=1)[0] key["experiment_id"] = 1000 key["experiment_date"] = "2018-10-30" self.experiment.insert1(key, allow_direct_insert=True) - @pytest.mark.skip(reason="temp") - def test_multi_processing(self): + def test_multi_processing(self, schema_any): assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" self.experiment.populate(processes=2) @@ -114,8 +107,7 @@ def test_multi_processing(self): == len(self.subject) * self.experiment.fake_experiments_per_subject ) - @pytest.mark.skip(reason="temp") - def test_max_multi_processing(self): + def test_max_multi_processing(self, schema_any): assert self.subject, "root tables are empty" assert not self.experiment, "table already filled?" self.experiment.populate(processes=None) @@ -124,8 +116,7 @@ def test_max_multi_processing(self): == len(self.subject) * self.experiment.fake_experiments_per_subject ) - @pytest.mark.skip(reason="temp") - def test_allow_insert(self): + def test_allow_insert(self, schema_any): assert self.subject, "root tables are empty" key = self.subject.fetch("KEY")[0] key["experiment_id"] = 1001 @@ -133,7 +124,6 @@ def test_allow_insert(self): with pytest.raises(DataJointError): self.experiment.insert1(key) - @pytest.mark.skip(reason="temp") def test_load_dependencies(self): schema = dj.Schema(f"{PREFIX}_load_dependencies_populate") From 57cddd5293e3b786fdf7b2a9445ee6e7aafd7d33 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 15:19:01 -0600 Subject: [PATCH 07/10] Switch from deprecated Version classes --- tests/conftest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6861214a..9b43c2eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,6 @@ import minio import urllib3 import certifi -from distutils.version import LooseVersion import shutil import pytest import networkx as nx @@ -68,9 +67,9 @@ def connection_root(connection_root_bare): dj.config["safemode"] = False conn_root = connection_root_bare # Create MySQL users - if LooseVersion(conn_root.query("select @@version;").fetchone()[0]) >= LooseVersion( - "8.0.0" - ): + if version.parse( + connection_root.query("select @@version;").fetchone()[0] + ) >= version.parse("8.0.0"): # create user if necessary on mysql8 conn_root.query( """ From 0de5a69fda533f70a702c3bb33933f0c4512f854 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 21:19:13 +0000 Subject: [PATCH 08/10] Format with black --- tests/test_autopopulate.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index d72cbc2c..c04827f5 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -23,7 +23,11 @@ def setup_class(cls): def teardown_class(cls): """Delete automatic tables just in case""" for autopop_table in ( - cls.channel, cls.ephys, cls.trial.Condition, cls.trial, cls.experiment + cls.channel, + cls.ephys, + cls.trial.Condition, + cls.trial, + cls.experiment, ): try: autopop_table.delete_quick() @@ -89,7 +93,10 @@ def test_populate_exclude_error_and_ignore_jobs(self, schema_any): schema_any.jobs.error(self.experiment.table_name, key, "") self.experiment.populate(reserve_jobs=True) - assert len(self.experiment.key_source & self.experiment) == len(self.experiment.key_source) - 2 + assert ( + len(self.experiment.key_source & self.experiment) + == len(self.experiment.key_source) - 2 + ) def test_allow_direct_insert(self, schema_any): assert self.subject, "root tables are empty" From 209a69b476b7ab74b0618c5d6cb6d14582642ae3 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 15:42:51 -0600 Subject: [PATCH 09/10] Fix syntax error --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9b43c2eb..0b146524 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,7 +68,7 @@ def connection_root(connection_root_bare): conn_root = connection_root_bare # Create MySQL users if version.parse( - connection_root.query("select @@version;").fetchone()[0] + conn_root.query("select @@version;").fetchone()[0] ) >= version.parse("8.0.0"): # create user if necessary on mysql8 conn_root.query( From 61c7e17cf885b74a37e7ec34b0b54b1eb603995f Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 16:01:09 -0600 Subject: [PATCH 10/10] Catch a DJ error on drop_quick --- tests/test_autopopulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index c04827f5..25f8e16e 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -31,7 +31,7 @@ def teardown_class(cls): ): try: autopop_table.delete_quick() - except pymysql.err.OperationalError: + except (pymysql.err.OperationalError, dj.errors.MissingTableError): # Table doesn't exist pass