diff --git a/PYTHON_API.md b/PYTHON_API.md new file mode 100644 index 00000000..7451c62f --- /dev/null +++ b/PYTHON_API.md @@ -0,0 +1,43 @@ +# API Functions + +Functions which make interacting with the models easier. + +## `create_document_file` + +Creates a document and file + +### Parameters + +- `user` The user creating the document and file. +- `group` The group creating the document and file. +- `category` models.Category, +- `document_title` Name of the document +- `file_name` Name of the file +- `file_content` File object +- `mime_type` File mime type +- `file_size` File size +- `additional_document_attributes` A dictionary containing the optional fields for the document. +- `additional_file_attributes` A dictionary containing the optional fields for the file. + +### Return + +- A tuple containing the created document and file. + +## `create_file` + +Creates a file with thumbnail + +### Parameters + +- `document` The document associated with the file. +- `user` The user who created the file. +- `group` The group who created the file. +- `name` Name of the file +- `content` File object +- `mime_type` File mime type +- `size` File size +- `additional_attributes` Kwargs containing the optional fields for the file. + +### Return + +- The created file object. diff --git a/README.md b/README.md index 7b562e72..2355e985 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ To load a set of categories run the following command: make load_example_data ``` +### Usage + +If alexandria is installed as a package you can use [PYTHON_API](PYTHON_API.md) for interacting with the models in an easier way. + ### Configuration Alexandria is a [12factor app](https://12factor.net/) which means that configuration is stored in environment variables. diff --git a/alexandria/core/api.py b/alexandria/core/api.py new file mode 100644 index 00000000..e2475e20 --- /dev/null +++ b/alexandria/core/api.py @@ -0,0 +1,96 @@ +import logging + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.core.files import File + +from alexandria.core import models + +log = logging.getLogger(__name__) + + +def create_document_file( + user: str, + group: str, + category: models.Category, + document_title: str, + file_name: str, + file_content: File, + mime_type: str, + file_size: int, + additional_document_attributes={}, + additional_file_attributes={}, +): + """ + Create a document and file with the given attributes. + + This function eases the creation of documents and files by automatically setting important fields. + Uses `create_file` to create the file. + """ + document = models.Document.objects.create( + title=document_title, + category=category, + created_by_user=user, + created_by_group=group, + modified_by_user=user, + modified_by_group=group, + **additional_document_attributes, + ) + file = create_file( + document=document, + user=user, + group=group, + name=file_name, + content=file_content, + mime_type=mime_type, + size=file_size, + **additional_file_attributes, + ) + + return document, file + + +def create_file( + document: models.Document, + user: str, + group: str, + name: str, + content: File, + mime_type: str, + size: int, + **additional_attributes +): + """ + Create a file with defaults and generate a thumbnail. + + Use this instead of the normal File.objects.create to ensure that all important fields are set. + As well as generating a thumbnail for the file. + """ + file = models.File.objects.create( + name=name, + content=content, + mime_type=mime_type, + size=size, + document=document, + encryption_status=( + settings.ALEXANDRIA_ENCRYPTION_METHOD + if settings.ALEXANDRIA_ENABLE_AT_REST_ENCRYPTION + else None + ), + created_by_user=user, + created_by_group=group, + modified_by_user=user, + modified_by_group=group, + **additional_attributes, + ) + + try: + file.create_thumbnail() + except ValidationError as e: # pragma: no cover + log.error( + "Object {obj} created successfully. Thumbnail creation failed. Error: {error}".format( + obj=file, error=e.messages + ) + ) + + return file diff --git a/alexandria/core/tests/__snapshots__/test_api.ambr b/alexandria/core/tests/__snapshots__/test_viewsets.ambr similarity index 100% rename from alexandria/core/tests/__snapshots__/test_api.ambr rename to alexandria/core/tests/__snapshots__/test_viewsets.ambr diff --git a/alexandria/core/tests/test_api.py b/alexandria/core/tests/test_api.py index 0c401828..52df1d95 100644 --- a/alexandria/core/tests/test_api.py +++ b/alexandria/core/tests/test_api.py @@ -1,252 +1,23 @@ -"""Module to test api in a generic way.""" - -import hashlib -import io -import json -import re -import uuid - -import inflection -import pytest -from django.db import connection -from django.db.models.fields.related import ManyToManyDescriptor -from django.test.utils import CaptureQueriesContext -from django.urls import reverse -from rest_framework_json_api.renderers import JSONRenderer -from syrupy.matchers import path_type - -from ..views import ( - CategoryViewSet, - DocumentViewSet, - FileViewSet, - MarkViewSet, - TagViewSet, -) - - -@pytest.fixture() -def deterministic_uuids(mocker): - # md5 hex digests are the same length as UUIDs, so django happily accepts - # them. Note also this has no cryptographic use here, so md5 is totally - # fine - hashy = hashlib.md5() - - def next_uuid(): - # add some string to the hash. This modifies it enough to yield a - # totally different value - hashy.update(b"x") - # format: 9336ebf2-5087-d91c-818e-e6e9 ec29 f8c1 - # lengths: 8 4 4 4 12 - digest = hashy.hexdigest() - return uuid.UUID( - "-".join( - [ - digest[:8], # 8 - digest[8:12], # 4 - digest[12:16], # 4 - digest[16:20], # 4 - digest[20:], # 4 - ] - ) - ) - - mocker.patch("uuid.uuid4", next_uuid) - - -@pytest.fixture( - params=[ - # add your viewset and expected queries run for generic api testing... - CategoryViewSet, - DocumentViewSet, - FileViewSet, - TagViewSet, - MarkViewSet, - ] -) -def viewset(request): - """ - Viewset fixture listing viewsets for generic testing. - - For generic testing viewset needs to meet following requirements: - * class fields `serializer_class` and `queryset` - * registered factory for model it serves - * registered uri with `SimpleRouter` - """ - viewset_instance = request.param() - model = viewset_instance.queryset.model - factory_name = inflection.underscore(model._meta.object_name) - base_name = model._meta.object_name.lower() - - viewset_instance.factory_name = factory_name - viewset_instance.base_name = base_name - viewset_instance.kwargs = {} - - return viewset_instance - - -@pytest.fixture -def fixture( - deterministic_uuids, - django_db_reset_sequences, - request, - viewset, -): - """Get fixture and many to many relations of given viewset.""" - fixture = request.getfixturevalue(viewset.factory_name) - included = ( - viewset.serializer_class.included_serializers - if hasattr(viewset.serializer_class, "included_serializers") - else {} - ) - for name in sorted(included.keys()): - relation_type = getattr(fixture.__class__, name) - # pytest factory boy doesn't have native ManyToMany support - # so needs to be handled manually - if isinstance(relation_type, ManyToManyDescriptor): - print("{0}_{1}".format(viewset.factory_name, name)) - request.getfixturevalue("{0}_{1}".format(viewset.factory_name, name)) - - return fixture - - -def prefetch_query_in_normalizer(query): - """ - Normalize `IN` queries. - - Using `prefetch_related()` leads to `IN` queries where we have no control over the - order of the parameters. Make sure the order in the snapshot is always - alphabetical. - """ - regex = r".* IN \((.*)\).*" - match = re.match(regex, query) - if match and match.groups(): - for group in match.groups(): - lst = group.split(", ") - new_string = ", ".join(sorted(lst)) - query = query.replace(group, new_string) - return query - - -def assert_response( - response, query_context, snapshot, *, request_payload=None, include_json=True -): - value = { - "status": response.status_code, - "query_count": len(query_context.captured_queries), - "queries": [ - prefetch_query_in_normalizer(query["sql"]) - for query in query_context.captured_queries - ], - "request": { - k: v for k, v in response.request.items() if not k.startswith("wsgi") - }, - "request_payload": request_payload if request_payload else None, - } - - # Drop `SAVEPOINT` statements because they will change on every run. - value["queries"] = list( - filter(lambda lst: "SAVEPOINT" not in lst, value["queries"]) - ) - - if include_json: - value["response"] = response.json() - - assert value == snapshot( - matcher=path_type( - { - ".*webdav-url": (str,), - }, - regex=True, - ) +from django.core.files.uploadedfile import SimpleUploadedFile + +from alexandria.core import api +from alexandria.core.factories import FileData + + +def test_create_document_file(db, category): + doc, file = api.create_document_file( + "Foo", + "Baz", + category, + "Bar.pdf", + "Mee.pdf", + SimpleUploadedFile( + name="test.png", + content=FileData.png, + content_type="png", + ), + "image/png", + 1, ) - - -@pytest.mark.freeze_time("2017-05-21") -def test_api_list(fixture, request, admin_client, snapshot, viewset, manabi): - url = reverse("{0}-list".format(viewset.base_name)) - - # create more data for proper num queries check - request.getfixturevalue(viewset.factory_name + "_factory").create_batch(2) - - included = getattr(viewset.serializer_class, "included_serializers", {}) - with CaptureQueriesContext(connection) as context: - response = admin_client.get(url, data={"include": ",".join(included.keys())}) - - assert_response(response, context, snapshot) - - -@pytest.mark.freeze_time("2017-05-21") -def test_api_detail(fixture, admin_client, viewset, snapshot, manabi): - url = reverse("{0}-detail".format(viewset.base_name), args=[fixture.pk]) - - included = getattr(viewset.serializer_class, "included_serializers", {}) - with CaptureQueriesContext(connection) as context: - response = admin_client.get(url, data={"include": ",".join(included.keys())}) - - if viewset.get_view_name() == "File": - content = response.json() - content["data"]["attributes"]["content"] = io.BytesIO(b"somefilecontent") - response.content = content - - assert_response(response, context, snapshot) - - -@pytest.mark.freeze_time("2017-05-21") -def test_api_create(fixture, admin_client, viewset, snapshot, manabi): - url = reverse("{0}-list".format(viewset.base_name)) - - serializer = viewset.serializer_class(fixture) - renderer = JSONRenderer() - context = {"view": viewset} - serializer_data = serializer.data - data = json.loads(renderer.render(serializer_data, renderer_context=context)) - fixture.delete() # avoid constraint issues - - opts = {} - - if viewset.get_view_name() == "File": - data = { - "content": io.BytesIO(b"FiLeCoNtEnt"), - "name": serializer.data["name"], - "variant": fixture.Variant.ORIGINAL.value, - "document": str(fixture.document.pk), - } - opts = {"format": "multipart"} - - if viewset.base_name in ["category"]: - with pytest.raises(NotImplementedError): - response = admin_client.post(url, data=data) - else: - with CaptureQueriesContext(connection) as context: - response = admin_client.post(url, data=data, **opts) - - assert_response(response, context, snapshot, request_payload=data) - - -@pytest.mark.freeze_time("2017-05-21") -def test_api_patch(fixture, admin_client, viewset, snapshot): - url = reverse("{0}-detail".format(viewset.base_name), args=[fixture.pk]) - if viewset.get_view_name() == "File": - return - serializer = viewset.serializer_class(fixture) - renderer = JSONRenderer() - context = {"view": viewset} - data = json.loads(renderer.render(serializer.data, renderer_context=context)) - - with CaptureQueriesContext(connection) as context: - response = admin_client.patch(url, data=data) - assert_response(response, context, snapshot, request_payload=data) - - -@pytest.mark.freeze_time("2017-05-21") -def test_api_destroy(fixture, admin_client, snapshot, viewset): - url = reverse("{0}-detail".format(viewset.base_name), args=[fixture.pk]) - - if viewset.base_name in ["category"]: - with pytest.raises(NotImplementedError): - response = admin_client.delete(url) - else: - with CaptureQueriesContext(connection) as context: - response = admin_client.delete(url) - - assert_response(response, context, snapshot, include_json=False) + assert doc.title == "Bar.pdf" + assert file.name == "Mee.pdf" diff --git a/alexandria/core/tests/test_viewsets.py b/alexandria/core/tests/test_viewsets.py new file mode 100644 index 00000000..0c401828 --- /dev/null +++ b/alexandria/core/tests/test_viewsets.py @@ -0,0 +1,252 @@ +"""Module to test api in a generic way.""" + +import hashlib +import io +import json +import re +import uuid + +import inflection +import pytest +from django.db import connection +from django.db.models.fields.related import ManyToManyDescriptor +from django.test.utils import CaptureQueriesContext +from django.urls import reverse +from rest_framework_json_api.renderers import JSONRenderer +from syrupy.matchers import path_type + +from ..views import ( + CategoryViewSet, + DocumentViewSet, + FileViewSet, + MarkViewSet, + TagViewSet, +) + + +@pytest.fixture() +def deterministic_uuids(mocker): + # md5 hex digests are the same length as UUIDs, so django happily accepts + # them. Note also this has no cryptographic use here, so md5 is totally + # fine + hashy = hashlib.md5() + + def next_uuid(): + # add some string to the hash. This modifies it enough to yield a + # totally different value + hashy.update(b"x") + # format: 9336ebf2-5087-d91c-818e-e6e9 ec29 f8c1 + # lengths: 8 4 4 4 12 + digest = hashy.hexdigest() + return uuid.UUID( + "-".join( + [ + digest[:8], # 8 + digest[8:12], # 4 + digest[12:16], # 4 + digest[16:20], # 4 + digest[20:], # 4 + ] + ) + ) + + mocker.patch("uuid.uuid4", next_uuid) + + +@pytest.fixture( + params=[ + # add your viewset and expected queries run for generic api testing... + CategoryViewSet, + DocumentViewSet, + FileViewSet, + TagViewSet, + MarkViewSet, + ] +) +def viewset(request): + """ + Viewset fixture listing viewsets for generic testing. + + For generic testing viewset needs to meet following requirements: + * class fields `serializer_class` and `queryset` + * registered factory for model it serves + * registered uri with `SimpleRouter` + """ + viewset_instance = request.param() + model = viewset_instance.queryset.model + factory_name = inflection.underscore(model._meta.object_name) + base_name = model._meta.object_name.lower() + + viewset_instance.factory_name = factory_name + viewset_instance.base_name = base_name + viewset_instance.kwargs = {} + + return viewset_instance + + +@pytest.fixture +def fixture( + deterministic_uuids, + django_db_reset_sequences, + request, + viewset, +): + """Get fixture and many to many relations of given viewset.""" + fixture = request.getfixturevalue(viewset.factory_name) + included = ( + viewset.serializer_class.included_serializers + if hasattr(viewset.serializer_class, "included_serializers") + else {} + ) + for name in sorted(included.keys()): + relation_type = getattr(fixture.__class__, name) + # pytest factory boy doesn't have native ManyToMany support + # so needs to be handled manually + if isinstance(relation_type, ManyToManyDescriptor): + print("{0}_{1}".format(viewset.factory_name, name)) + request.getfixturevalue("{0}_{1}".format(viewset.factory_name, name)) + + return fixture + + +def prefetch_query_in_normalizer(query): + """ + Normalize `IN` queries. + + Using `prefetch_related()` leads to `IN` queries where we have no control over the + order of the parameters. Make sure the order in the snapshot is always + alphabetical. + """ + regex = r".* IN \((.*)\).*" + match = re.match(regex, query) + if match and match.groups(): + for group in match.groups(): + lst = group.split(", ") + new_string = ", ".join(sorted(lst)) + query = query.replace(group, new_string) + return query + + +def assert_response( + response, query_context, snapshot, *, request_payload=None, include_json=True +): + value = { + "status": response.status_code, + "query_count": len(query_context.captured_queries), + "queries": [ + prefetch_query_in_normalizer(query["sql"]) + for query in query_context.captured_queries + ], + "request": { + k: v for k, v in response.request.items() if not k.startswith("wsgi") + }, + "request_payload": request_payload if request_payload else None, + } + + # Drop `SAVEPOINT` statements because they will change on every run. + value["queries"] = list( + filter(lambda lst: "SAVEPOINT" not in lst, value["queries"]) + ) + + if include_json: + value["response"] = response.json() + + assert value == snapshot( + matcher=path_type( + { + ".*webdav-url": (str,), + }, + regex=True, + ) + ) + + +@pytest.mark.freeze_time("2017-05-21") +def test_api_list(fixture, request, admin_client, snapshot, viewset, manabi): + url = reverse("{0}-list".format(viewset.base_name)) + + # create more data for proper num queries check + request.getfixturevalue(viewset.factory_name + "_factory").create_batch(2) + + included = getattr(viewset.serializer_class, "included_serializers", {}) + with CaptureQueriesContext(connection) as context: + response = admin_client.get(url, data={"include": ",".join(included.keys())}) + + assert_response(response, context, snapshot) + + +@pytest.mark.freeze_time("2017-05-21") +def test_api_detail(fixture, admin_client, viewset, snapshot, manabi): + url = reverse("{0}-detail".format(viewset.base_name), args=[fixture.pk]) + + included = getattr(viewset.serializer_class, "included_serializers", {}) + with CaptureQueriesContext(connection) as context: + response = admin_client.get(url, data={"include": ",".join(included.keys())}) + + if viewset.get_view_name() == "File": + content = response.json() + content["data"]["attributes"]["content"] = io.BytesIO(b"somefilecontent") + response.content = content + + assert_response(response, context, snapshot) + + +@pytest.mark.freeze_time("2017-05-21") +def test_api_create(fixture, admin_client, viewset, snapshot, manabi): + url = reverse("{0}-list".format(viewset.base_name)) + + serializer = viewset.serializer_class(fixture) + renderer = JSONRenderer() + context = {"view": viewset} + serializer_data = serializer.data + data = json.loads(renderer.render(serializer_data, renderer_context=context)) + fixture.delete() # avoid constraint issues + + opts = {} + + if viewset.get_view_name() == "File": + data = { + "content": io.BytesIO(b"FiLeCoNtEnt"), + "name": serializer.data["name"], + "variant": fixture.Variant.ORIGINAL.value, + "document": str(fixture.document.pk), + } + opts = {"format": "multipart"} + + if viewset.base_name in ["category"]: + with pytest.raises(NotImplementedError): + response = admin_client.post(url, data=data) + else: + with CaptureQueriesContext(connection) as context: + response = admin_client.post(url, data=data, **opts) + + assert_response(response, context, snapshot, request_payload=data) + + +@pytest.mark.freeze_time("2017-05-21") +def test_api_patch(fixture, admin_client, viewset, snapshot): + url = reverse("{0}-detail".format(viewset.base_name), args=[fixture.pk]) + if viewset.get_view_name() == "File": + return + serializer = viewset.serializer_class(fixture) + renderer = JSONRenderer() + context = {"view": viewset} + data = json.loads(renderer.render(serializer.data, renderer_context=context)) + + with CaptureQueriesContext(connection) as context: + response = admin_client.patch(url, data=data) + assert_response(response, context, snapshot, request_payload=data) + + +@pytest.mark.freeze_time("2017-05-21") +def test_api_destroy(fixture, admin_client, snapshot, viewset): + url = reverse("{0}-detail".format(viewset.base_name), args=[fixture.pk]) + + if viewset.base_name in ["category"]: + with pytest.raises(NotImplementedError): + response = admin_client.delete(url) + else: + with CaptureQueriesContext(connection) as context: + response = admin_client.delete(url) + + assert_response(response, context, snapshot, include_json=False) diff --git a/alexandria/core/views.py b/alexandria/core/views.py index 8c247d8b..fa1f3e7b 100644 --- a/alexandria/core/views.py +++ b/alexandria/core/views.py @@ -38,6 +38,7 @@ ) from . import models, serializers +from .api import create_document_file from .filters import ( CategoryFilterSet, DocumentFilterSet, @@ -136,31 +137,27 @@ def convert(self, request, pk=None): request.user, settings.ALEXANDRIA_CREATED_BY_GROUP_PROPERTY, None ) - converted_document = models.Document.objects.create( - title={k: f"{ splitext(v)[0]}.pdf" for k, v in document.title.items()}, - description=document.description, - category=document.category, - date=document.date, - metainfo=document.metainfo, - created_by_user=username, - created_by_group=group, - modified_by_user=username, - modified_by_group=group, - ) file_name = f"{splitext(file.name)[0]}.pdf" - converted_file = models.File.objects.create( - document=converted_document, - name=file_name, - content=ContentFile(response.content, file_name), + converted_document, __ = create_document_file( + user=username, + group=group, + category=document.category, + document_title={ + k: f"{ splitext(v)[0]}.pdf" for k, v in document.title.items() + }, + file_name=file_name, + file_content=ContentFile(response.content, file_name), mime_type="application/pdf", - size=len(response.content), - metainfo=file.metainfo, - created_by_user=username, - created_by_group=group, - modified_by_user=username, - modified_by_group=group, + file_size=len(response.content), + additional_document_attributes={ + "description": document.description, + "date": document.date, + "metainfo": document.metainfo, + }, + additional_file_attributes={ + "metainfo": file.metainfo, + }, ) - converted_file.create_thumbnail() serializer = self.get_serializer(converted_document) headers = self.get_success_headers(serializer.data)