Merge pull request #1118 from ethho/dev-tests-plat-165

PLAT-165: Migrate test_s3.py

tests/conftest.py

@@ -23,6 +23,7 @@
     schema_simple,
     schema_advanced,
     schema_adapted,
+    schema_external,
 )
 
 
@@ -38,6 +39,20 @@ def monkeymodule():
         yield mp
 
 
+@pytest.fixture
+def enable_adapted_types(monkeypatch):
+    monkeypatch.setenv(ADAPTED_TYPE_SWITCH, "TRUE")
+    yield
+    monkeypatch.delenv(ADAPTED_TYPE_SWITCH, raising=True)
+
+
+@pytest.fixture
+def enable_filepath_feature(monkeypatch):
+    monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, "TRUE")
+    yield
+    monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True)
+
+
 @pytest.fixture(scope="session")
 def connection_root_bare():
     connection = dj.Connection(
@@ -161,6 +176,24 @@ def connection_test(connection_root):
     connection.close()
 
 
+@pytest.fixture(scope="session")
+def stores_config():
+    stores_config = {
+        "raw": dict(protocol="file", location=tempfile.mkdtemp()),
+        "repo": dict(
+            stage=tempfile.mkdtemp(), protocol="file", location=tempfile.mkdtemp()
+        ),
+        "repo-s3": dict(
+            S3_CONN_INFO, protocol="s3", location="dj/repo", stage=tempfile.mkdtemp()
+        ),
+        "local": dict(protocol="file", location=tempfile.mkdtemp(), subfolding=(1, 1)),
+        "share": dict(
+            S3_CONN_INFO, protocol="s3", location="dj/store/repo", subfolding=(2, 4)
+        ),
+    }
+    return stores_config
+
+
 @pytest.fixture
 def schema_any(connection_test):
     schema_any = dj.Schema(
@@ -254,6 +287,28 @@ def schema_adv(connection_test):
     schema.drop()
 
 
+@pytest.fixture
+def schema_ext(connection_test, stores_config, enable_filepath_feature):
+    schema = dj.Schema(
+        PREFIX + "_extern",
+        context=schema_external.LOCALS_EXTERNAL,
+        connection=connection_test,
+    )
+    dj.config["stores"] = stores_config
+    dj.config["cache"] = tempfile.mkdtemp()
+
+    schema(schema_external.Simple)
+    schema(schema_external.SimpleRemote)
+    schema(schema_external.Seed)
+    schema(schema_external.Dimension)
+    schema(schema_external.Image)
+    schema(schema_external.Attach)
+    schema(schema_external.Filepath)
+    schema(schema_external.FilepathS3)
+    yield schema
+    schema.drop()
+
+
 @pytest.fixture(scope="session")
 def http_client():
     # Initialize httpClient with relevant timeout.
@@ -270,6 +325,7 @@ def http_client():
 
 @pytest.fixture(scope="session")
 def minio_client_bare(http_client):
+    """Initialize MinIO with an endpoint and access/secret keys."""
     client = minio.Minio(
         S3_CONN_INFO["endpoint"],
         access_key=S3_CONN_INFO["access_key"],
@@ -282,8 +338,8 @@ def minio_client_bare(http_client):
 
 @pytest.fixture(scope="session")
 def minio_client(minio_client_bare):
-    """Initialize MinIO with an endpoint and access/secret keys."""
-    # Bootstrap MinIO bucket
+    """Initialize a MinIO client and create buckets for testing session."""
+    # Setup MinIO bucket
     aws_region = "us-east-1"
     try:
         minio_client_bare.make_bucket(S3_CONN_INFO["bucket"], location=aws_region)

tests/schema_external.py

@@ -0,0 +1,89 @@
+"""
+A schema for testing external attributes
+"""
+
+import tempfile
+import inspect
+import datajoint as dj
+from . import PREFIX, CONN_INFO, S3_CONN_INFO
+import numpy as np
+
+
+class Simple(dj.Manual):
+    definition = """
+    simple  : int
+    ---
+    item  : blob@local
+    """
+
+
+class SimpleRemote(dj.Manual):
+    definition = """
+    simple  : int
+    ---
+    item  : blob@share
+    """
+
+
+class Seed(dj.Lookup):
+    definition = """
+    seed :  int
+    """
+    contents = zip(range(4))
+
+
+class Dimension(dj.Lookup):
+    definition = """
+    dim  : int
+    ---
+    dimensions  : blob
+    """
+    contents = ([0, [100, 50]], [1, [3, 4, 8, 6]])
+
+
+class Image(dj.Computed):
+    definition = """
+    # table for storing
+    -> Seed
+    -> Dimension
+    ----
+    img : blob@share     #  objects are stored as specified by dj.config['stores']['share']
+    neg : blob@local   # objects are stored as specified by dj.config['stores']['local']
+    """
+
+    def make(self, key):
+        np.random.seed(key["seed"])
+        img = np.random.rand(*(Dimension() & key).fetch1("dimensions"))
+        self.insert1(dict(key, img=img, neg=-img.astype(np.float32)))
+
+
+class Attach(dj.Manual):
+    definition = """
+    # table for storing attachments
+    attach : int
+    ----
+    img : attach@share    #  attachments are stored as specified by: dj.config['stores']['raw']
+    txt : attach      #  attachments are stored directly in the database
+    """
+
+
+class Filepath(dj.Manual):
+    definition = """
+    # table for file management
+    fnum : int # test comment containing :
+    ---
+    img : filepath@repo  # managed files
+    """
+
+
+class FilepathS3(dj.Manual):
+    definition = """
+    # table for file management
+    fnum : int
+    ---
+    img : filepath@repo-s3  # managed files
+    """
+
+
+LOCALS_EXTERNAL = {k: v for k, v in locals().items() if inspect.isclass(v)}
+__all__ = list(LOCALS_EXTERNAL)

tests/test_adapted_attributes.py

@@ -2,7 +2,6 @@
 import pytest
 import tempfile
 import datajoint as dj
-from datajoint.errors import ADAPTED_TYPE_SWITCH, FILEPATH_FEATURE_SWITCH
 import networkx as nx
 from itertools import zip_longest
 from . import schema_adapted
@@ -17,20 +16,6 @@ def adapted_graph_instance():
     yield schema_adapted.GraphAdapter()
 
 
-@pytest.fixture
-def enable_adapted_types(monkeypatch):
-    monkeypatch.setenv(ADAPTED_TYPE_SWITCH, "TRUE")
-    yield
-    monkeypatch.delenv(ADAPTED_TYPE_SWITCH, raising=True)
-
-
-@pytest.fixture
-def enable_filepath_feature(monkeypatch):
-    monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, "TRUE")
-    yield
-    monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True)
-
-
 @pytest.fixture
 def schema_ad(
     connection_test,

tests/test_s3.py

@@ -0,0 +1,50 @@
+import pytest
+import urllib3
+import certifi
+from .schema_external import SimpleRemote
+from datajoint.errors import DataJointError
+from datajoint.hash import uuid_from_buffer
+from datajoint.blob import pack
+from . import S3_CONN_INFO
+from minio import Minio
+
+
+class TestS3:
+    def test_connection(self, http_client, minio_client):
+        assert minio_client.bucket_exists(S3_CONN_INFO["bucket"])
+
+    def test_connection_secure(self, minio_client):
+        assert minio_client.bucket_exists(S3_CONN_INFO["bucket"])
+
+    def test_remove_object_exception(self, schema_ext):
+        # https://github.com/datajoint/datajoint-python/issues/952
+
+        # Insert some test data and remove it so that the external table is populated
+        test = [1, [1, 2, 3]]
+        SimpleRemote.insert1(test)
+        SimpleRemote.delete()
+
+        # Save the old external table minio client
+        old_client = schema_ext.external["share"].s3.client
+
+        # Apply our new minio client which has a user that does not exist
+        schema_ext.external["share"].s3.client = Minio(
+            S3_CONN_INFO["endpoint"],
+            access_key="jeffjeff",
+            secret_key="jeffjeff",
+            secure=False,
+        )
+
+        # This method returns a list of errors
+        error_list = schema_ext.external["share"].delete(
+            delete_external_files=True, errors_as_string=False
+        )
+
+        # Teardown
+        schema_ext.external["share"].s3.client = old_client
+        schema_ext.external["share"].delete(delete_external_files=True)
+
+        with pytest.raises(DataJointError):
+            # Raise the error we want if the error matches the expected uuid
+            if str(error_list[0][0]) == str(uuid_from_buffer(pack(test[1]))):
+                raise error_list[0][2]