From 27fd65c904053795cb739ac99d8904ab703aad81 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Mon, 30 Sep 2024 16:41:17 +0200 Subject: [PATCH 01/10] prep editor types --- mex/backend/graph/connector.py | 16 +-- .../cypher/fetch_extracted_or_rule_items.cql | 2 +- .../graph/cypher/fetch_merged_items.cql | 2 +- mex/backend/graph/cypher/merge_edges.cql | 4 +- mex/backend/graph/transform.py | 25 ----- mex/backend/ingest/main.py | 3 + mex/backend/main.py | 7 +- pdm.lock | 104 +++++++++--------- pyproject.toml | 2 +- tests/graph/test_query.py | 8 +- tests/graph/test_transform.py | 18 +-- 11 files changed, 75 insertions(+), 116 deletions(-) diff --git a/mex/backend/graph/connector.py b/mex/backend/graph/connector.py index 947e7a5b..471cf09e 100644 --- a/mex/backend/graph/connector.py +++ b/mex/backend/graph/connector.py @@ -3,7 +3,7 @@ from string import Template from typing import Annotated, Any, Literal, cast -from neo4j import Driver, GraphDatabase +from neo4j import Driver, GraphDatabase, NotificationMinimumSeverity from pydantic import Field from mex.backend.fields import ( @@ -17,7 +17,7 @@ ) from mex.backend.graph.models import Result from mex.backend.graph.query import QueryBuilder -from mex.backend.graph.transform import expand_references_in_search_result, to_primitive +from mex.backend.graph.transform import expand_references_in_search_result from mex.backend.settings import BackendSettings from mex.common.connector import BaseConnector from mex.common.exceptions import MExError @@ -84,6 +84,7 @@ def _init_driver(self) -> Driver: settings.graph_password.get_secret_value(), ), database=settings.graph_db, + warn_notification_severity=NotificationMinimumSeverity.OFF, ) def _check_connectivity_and_authentication(self) -> Result: @@ -379,12 +380,12 @@ def _merge_item( mutable_fields = set(MUTABLE_FIELDS_BY_CLASS_NAME[model.entityType]) final_fields = set(FINAL_FIELDS_BY_CLASS_NAME[model.entityType]) - mutable_values = to_primitive(model, include=mutable_fields) - final_values = to_primitive(model, include=final_fields) + mutable_values = model.model_dump(include=mutable_fields) + final_values = model.model_dump(include=final_fields) all_values = {**mutable_values, **final_values} - text_values = to_primitive(model, include=text_fields) - link_values = to_primitive(model, include=link_fields) + text_values = model.model_dump(include=text_fields) + link_values = model.model_dump(include=link_fields) nested_edge_labels: list[str] = [] nested_node_labels: list[str] = [] @@ -445,7 +446,7 @@ def _merge_edges( query_builder = QueryBuilder.get() ref_fields = REFERENCE_FIELDS_BY_CLASS_NAME[model.entityType] - ref_values = to_primitive(model, include=set(ref_fields)) + ref_values = model.model_dump(include=set(ref_fields)) ref_values.update(extra_refs or {}) ref_labels: list[str] = [] @@ -464,7 +465,6 @@ def _merge_edges( merged_label=ensure_prefix(model.stemType, "Merged"), ref_labels=ref_labels, ) - return self.commit( query, **constraints, diff --git a/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql b/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql index fda0ab33..603cd4a3 100644 --- a/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql +++ b/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql @@ -59,7 +59,7 @@ CALL { } WITH n, collect(ref) as refs RETURN n{.*, entityType: head(labels(n)), _refs: refs} - ORDER BY n.identifier ASC + ORDER BY elementId(n) ASC SKIP $skip LIMIT $limit } diff --git a/mex/backend/graph/cypher/fetch_merged_items.cql b/mex/backend/graph/cypher/fetch_merged_items.cql index 4c4b6072..56a26305 100644 --- a/mex/backend/graph/cypher/fetch_merged_items.cql +++ b/mex/backend/graph/cypher/fetch_merged_items.cql @@ -60,7 +60,7 @@ CALL { } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, n.identifier ASC + ORDER BY merged.identifier, elementId(n) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip diff --git a/mex/backend/graph/cypher/merge_edges.cql b/mex/backend/graph/cypher/merge_edges.cql index 3d4253e3..7b1a82af 100644 --- a/mex/backend/graph/cypher/merge_edges.cql +++ b/mex/backend/graph/cypher/merge_edges.cql @@ -34,7 +34,7 @@ CALL { RETURN null as edge <%- endif %> } -WITH source, collect(edge) as edges +WITH source, count(edge) as merged, collect(edge) as edges CALL { WITH source, edges MATCH (source)-[outdated_edge]->(:<>) @@ -42,4 +42,4 @@ CALL { DELETE outdated_edge RETURN count(outdated_edge) as pruned } -RETURN count(edges) as merged, pruned, edges; +RETURN merged, pruned, edges; diff --git a/mex/backend/graph/transform.py b/mex/backend/graph/transform.py index 558e6492..ca6ccbbe 100644 --- a/mex/backend/graph/transform.py +++ b/mex/backend/graph/transform.py @@ -1,7 +1,5 @@ from typing import Any, TypedDict, cast -from pydantic import BaseModel - class _SearchResultReference(TypedDict): """Helper class to show the structure of search result references.""" @@ -24,26 +22,3 @@ def expand_references_in_search_result(item: dict[str, Any]) -> None: length_needed = 1 + ref["position"] - len(target_list) target_list.extend([None] * length_needed) target_list[ref["position"]] = ref["value"] - - -def to_primitive( - obj: BaseModel, - include: set[str] | None = None, - exclude: set[str] | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, -) -> Any: - """Convert model object into python primitives compatible with graph ingestion.""" - return obj.__pydantic_serializer__.to_python( - obj, - mode="json", - by_alias=by_alias, - include=include, - exclude=exclude, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - fallback=str, - ) diff --git a/mex/backend/ingest/main.py b/mex/backend/ingest/main.py index 3dca7a69..4130c565 100644 --- a/mex/backend/ingest/main.py +++ b/mex/backend/ingest/main.py @@ -12,4 +12,7 @@ def ingest_extracted_items(request: BulkIngestRequest) -> BulkIngestResponse: """Ingest batches of extracted items grouped by their type.""" connector = GraphConnector.get() identifiers = connector.ingest(request.items) + from mex.common.logging import logger + + logger.info(request.model_dump_json(indent=2)) return BulkIngestResponse(identifiers=identifiers) diff --git a/mex/backend/main.py b/mex/backend/main.py index eefcb14a..1aa5f307 100644 --- a/mex/backend/main.py +++ b/mex/backend/main.py @@ -22,11 +22,8 @@ from mex.backend.settings import BackendSettings from mex.common.cli import entrypoint from mex.common.connector import CONNECTOR_STORE -from mex.common.types import ( - EXTRACTED_IDENTIFIER_CLASSES, - MERGED_IDENTIFIER_CLASSES, - MEX_ID_PATTERN, -) +from mex.common.types import EXTRACTED_IDENTIFIER_CLASSES, MERGED_IDENTIFIER_CLASSES +from mex.common.types import IDENTIFIER_PATTERN as MEX_ID_PATTERN def create_openapi_schema() -> dict[str, Any]: diff --git a/pdm.lock b/pdm.lock index 991e6be0..1fed8b75 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:e8fb5e3cc5b8a0c4be780675b49c6f59a95f107aee5f02c5eb55a8736a898b5c" +content_hash = "sha256:5695fbba8be6dfc6f25fd616d36b3ea88aaff16159ba7269baed5730451fd0cf" [[metadata.targets]] requires_python = "==3.11.*" @@ -548,8 +548,8 @@ name = "mex-common" version = "0.36.1" requires_python = "<3.13,>=3.11" git = "https://github.com/robert-koch-institut/mex-common.git" -ref = "0.36.1" -revision = "29f3941499f2b0466a3ce5008d3350075df9d2d3" +ref = "feature/mx-1702-type-prep" +revision = "612c7cc4471eefd9b7ed1d89b7ff49f52d3a46d0" summary = "Common library for MEx python projects." groups = ["default"] marker = "python_version == \"3.11\"" @@ -615,7 +615,7 @@ files = [ [[package]] name = "neo4j" -version = "5.24.0" +version = "5.25.0" requires_python = ">=3.7" summary = "Neo4j Bolt driver for Python" groups = ["default"] @@ -624,8 +624,8 @@ dependencies = [ "pytz", ] files = [ - {file = "neo4j-5.24.0-py3-none-any.whl", hash = "sha256:5b4705cfe8130020f33e75e31ad3fcfe67ee958e07d0c3c4936e9c8245a1ea58"}, - {file = "neo4j-5.24.0.tar.gz", hash = "sha256:499ca35135847528f4ee70314bd49c8b08b031e4dfd588bb06c1c2fb35d729e2"}, + {file = "neo4j-5.25.0-py3-none-any.whl", hash = "sha256:df310eee9a4f9749fb32bb9f1aa68711ac417b7eba3e42faefd6848038345ffa"}, + {file = "neo4j-5.25.0.tar.gz", hash = "sha256:7c82001c45319092cc0b5df4c92894553b7ab97bd4f59655156fa9acab83aec9"}, ] [[package]] @@ -751,7 +751,7 @@ files = [ [[package]] name = "prompt-toolkit" -version = "3.0.47" +version = "3.0.48" requires_python = ">=3.7.0" summary = "Library for building powerful interactive command lines in Python" groups = ["dev"] @@ -760,8 +760,8 @@ dependencies = [ "wcwidth", ] files = [ - {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"}, - {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"}, + {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"}, + {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"}, ] [[package]] @@ -1020,30 +1020,30 @@ files = [ [[package]] name = "ruff" -version = "0.6.7" +version = "0.6.8" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." groups = ["dev"] marker = "python_version == \"3.11\"" files = [ - {file = "ruff-0.6.7-py3-none-linux_armv6l.whl", hash = "sha256:08277b217534bfdcc2e1377f7f933e1c7957453e8a79764d004e44c40db923f2"}, - {file = "ruff-0.6.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c6707a32e03b791f4448dc0dce24b636cbcdee4dd5607adc24e5ee73fd86c00a"}, - {file = "ruff-0.6.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:533d66b7774ef224e7cf91506a7dafcc9e8ec7c059263ec46629e54e7b1f90ab"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17a86aac6f915932d259f7bec79173e356165518859f94649d8c50b81ff087e9"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3f8822defd260ae2460ea3832b24d37d203c3577f48b055590a426a722d50ef"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ba4efe5c6dbbb58be58dd83feedb83b5e95c00091bf09987b4baf510fee5c99"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:525201b77f94d2b54868f0cbe5edc018e64c22563da6c5c2e5c107a4e85c1c0d"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8854450839f339e1049fdbe15d875384242b8e85d5c6947bb2faad33c651020b"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f0b62056246234d59cbf2ea66e84812dc9ec4540518e37553513392c171cb18"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b1462fa56c832dc0cea5b4041cfc9c97813505d11cce74ebc6d1aae068de36b"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:02b083770e4cdb1495ed313f5694c62808e71764ec6ee5db84eedd82fd32d8f5"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0c05fd37013de36dfa883a3854fae57b3113aaa8abf5dea79202675991d48624"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f49c9caa28d9bbfac4a637ae10327b3db00f47d038f3fbb2195c4d682e925b14"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a0e1655868164e114ba43a908fd2d64a271a23660195017c17691fb6355d59bb"}, - {file = "ruff-0.6.7-py3-none-win32.whl", hash = "sha256:a939ca435b49f6966a7dd64b765c9df16f1faed0ca3b6f16acdf7731969deb35"}, - {file = "ruff-0.6.7-py3-none-win_amd64.whl", hash = "sha256:590445eec5653f36248584579c06252ad2e110a5d1f32db5420de35fb0e1c977"}, - {file = "ruff-0.6.7-py3-none-win_arm64.whl", hash = "sha256:b28f0d5e2f771c1fe3c7a45d3f53916fc74a480698c4b5731f0bea61e52137c8"}, - {file = "ruff-0.6.7.tar.gz", hash = "sha256:44e52129d82266fa59b587e2cd74def5637b730a69c4542525dfdecfaae38bd5"}, + {file = "ruff-0.6.8-py3-none-linux_armv6l.whl", hash = "sha256:77944bca110ff0a43b768f05a529fecd0706aac7bcce36d7f1eeb4cbfca5f0f2"}, + {file = "ruff-0.6.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27b87e1801e786cd6ede4ada3faa5e254ce774de835e6723fd94551464c56b8c"}, + {file = "ruff-0.6.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cd48f945da2a6334f1793d7f701725a76ba93bf3d73c36f6b21fb04d5338dcf5"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:677e03c00f37c66cea033274295a983c7c546edea5043d0c798833adf4cf4c6f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f1476236b3eacfacfc0f66aa9e6cd39f2a624cb73ea99189556015f27c0bdeb"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f5a2f17c7d32991169195d52a04c95b256378bbf0de8cb98478351eb70d526f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5fd0d4b7b1457c49e435ee1e437900ced9b35cb8dc5178921dfb7d98d65a08d0"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8034b19b993e9601f2ddf2c517451e17a6ab5cdb1c13fdff50c1442a7171d87"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cfb227b932ba8ef6e56c9f875d987973cd5e35bc5d05f5abf045af78ad8e098"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef0411eccfc3909269fed47c61ffebdcb84a04504bafa6b6df9b85c27e813b0"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:007dee844738c3d2e6c24ab5bc7d43c99ba3e1943bd2d95d598582e9c1b27750"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ce60058d3cdd8490e5e5471ef086b3f1e90ab872b548814e35930e21d848c9ce"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1085c455d1b3fdb8021ad534379c60353b81ba079712bce7a900e834859182fa"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:70edf6a93b19481affd287d696d9e311388d808671bc209fb8907b46a8c3af44"}, + {file = "ruff-0.6.8-py3-none-win32.whl", hash = "sha256:792213f7be25316f9b46b854df80a77e0da87ec66691e8f012f887b4a671ab5a"}, + {file = "ruff-0.6.8-py3-none-win_amd64.whl", hash = "sha256:ec0517dc0f37cad14a5319ba7bba6e7e339d03fbf967a6d69b0907d61be7a263"}, + {file = "ruff-0.6.8-py3-none-win_arm64.whl", hash = "sha256:8d3bb2e3fbb9875172119021a13eed38849e762499e3cfde9588e4b4d70968dc"}, + {file = "ruff-0.6.8.tar.gz", hash = "sha256:a5bf44b1aa0adaf6d9d20f86162b34f7c593bfedabc51239953e446aefc8ce18"}, ] [[package]] @@ -1254,14 +1254,14 @@ files = [ [[package]] name = "tzdata" -version = "2024.1" +version = "2024.2" requires_python = ">=2" summary = "Provider of IANA time zone data" groups = ["default"] marker = "python_version == \"3.11\"" files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] [[package]] @@ -1278,7 +1278,7 @@ files = [ [[package]] name = "uvicorn" -version = "0.30.6" +version = "0.31.0" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["default"] @@ -1289,13 +1289,13 @@ dependencies = [ "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ - {file = "uvicorn-0.30.6-py3-none-any.whl", hash = "sha256:65fd46fe3fda5bdc1b03b94eb634923ff18cd35b2f084813ea79d1f103f711b5"}, - {file = "uvicorn-0.30.6.tar.gz", hash = "sha256:4b15decdda1e72be08209e860a1e10e92439ad5b97cf44cc945fcbee66fc5788"}, + {file = "uvicorn-0.31.0-py3-none-any.whl", hash = "sha256:cac7be4dd4d891c363cd942160a7b02e69150dcbc7a36be04d5f4af4b17c8ced"}, + {file = "uvicorn-0.31.0.tar.gz", hash = "sha256:13bc21373d103859f68fe739608e2eb054a816dea79189bc3ca08ea89a275906"}, ] [[package]] name = "uvicorn" -version = "0.30.6" +version = "0.31.0" extras = ["standard"] requires_python = ">=3.8" summary = "The lightning-fast ASGI server." @@ -1306,14 +1306,14 @@ dependencies = [ "httptools>=0.5.0", "python-dotenv>=0.13", "pyyaml>=5.1", - "uvicorn==0.30.6", + "uvicorn==0.31.0", "uvloop!=0.15.0,!=0.15.1,>=0.14.0; (sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"", "watchfiles>=0.13", "websockets>=10.4", ] files = [ - {file = "uvicorn-0.30.6-py3-none-any.whl", hash = "sha256:65fd46fe3fda5bdc1b03b94eb634923ff18cd35b2f084813ea79d1f103f711b5"}, - {file = "uvicorn-0.30.6.tar.gz", hash = "sha256:4b15decdda1e72be08209e860a1e10e92439ad5b97cf44cc945fcbee66fc5788"}, + {file = "uvicorn-0.31.0-py3-none-any.whl", hash = "sha256:cac7be4dd4d891c363cd942160a7b02e69150dcbc7a36be04d5f4af4b17c8ced"}, + {file = "uvicorn-0.31.0.tar.gz", hash = "sha256:13bc21373d103859f68fe739608e2eb054a816dea79189bc3ca08ea89a275906"}, ] [[package]] @@ -1376,23 +1376,23 @@ files = [ [[package]] name = "websockets" -version = "13.0.1" +version = "13.1" requires_python = ">=3.8" summary = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" groups = ["default"] marker = "python_version == \"3.11\"" files = [ - {file = "websockets-13.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:699ba9dd6a926f82a277063603fc8d586b89f4cb128efc353b749b641fcddda7"}, - {file = "websockets-13.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf2fae6d85e5dc384bf846f8243ddaa9197f3a1a70044f59399af001fd1f51d4"}, - {file = "websockets-13.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:52aed6ef21a0f1a2a5e310fb5c42d7555e9c5855476bbd7173c3aa3d8a0302f2"}, - {file = "websockets-13.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8eb2b9a318542153674c6e377eb8cb9ca0fc011c04475110d3477862f15d29f0"}, - {file = "websockets-13.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5df891c86fe68b2c38da55b7aea7095beca105933c697d719f3f45f4220a5e0e"}, - {file = "websockets-13.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fac2d146ff30d9dd2fcf917e5d147db037a5c573f0446c564f16f1f94cf87462"}, - {file = "websockets-13.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b8ac5b46fd798bbbf2ac6620e0437c36a202b08e1f827832c4bf050da081b501"}, - {file = "websockets-13.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:46af561eba6f9b0848b2c9d2427086cabadf14e0abdd9fde9d72d447df268418"}, - {file = "websockets-13.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b5a06d7f60bc2fc378a333978470dfc4e1415ee52f5f0fce4f7853eb10c1e9df"}, - {file = "websockets-13.0.1-cp311-cp311-win32.whl", hash = "sha256:556e70e4f69be1082e6ef26dcb70efcd08d1850f5d6c5f4f2bcb4e397e68f01f"}, - {file = "websockets-13.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:67494e95d6565bf395476e9d040037ff69c8b3fa356a886b21d8422ad86ae075"}, - {file = "websockets-13.0.1-py3-none-any.whl", hash = "sha256:b80f0c51681c517604152eb6a572f5a9378f877763231fddb883ba2f968e8817"}, - {file = "websockets-13.0.1.tar.gz", hash = "sha256:4d6ece65099411cfd9a48d13701d7438d9c34f479046b34c50ff60bb8834e43e"}, + {file = "websockets-13.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:61fc0dfcda609cda0fc9fe7977694c0c59cf9d749fbb17f4e9483929e3c48a19"}, + {file = "websockets-13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ceec59f59d092c5007e815def4ebb80c2de330e9588e101cf8bd94c143ec78a5"}, + {file = "websockets-13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1dca61c6db1166c48b95198c0b7d9c990b30c756fc2923cc66f68d17dc558fd"}, + {file = "websockets-13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:308e20f22c2c77f3f39caca508e765f8725020b84aa963474e18c59accbf4c02"}, + {file = "websockets-13.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d516c325e6540e8a57b94abefc3459d7dab8ce52ac75c96cad5549e187e3a7"}, + {file = "websockets-13.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87c6e35319b46b99e168eb98472d6c7d8634ee37750d7693656dc766395df096"}, + {file = "websockets-13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5f9fee94ebafbc3117c30be1844ed01a3b177bb6e39088bc6b2fa1dc15572084"}, + {file = "websockets-13.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7c1e90228c2f5cdde263253fa5db63e6653f1c00e7ec64108065a0b9713fa1b3"}, + {file = "websockets-13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6548f29b0e401eea2b967b2fdc1c7c7b5ebb3eeb470ed23a54cd45ef078a0db9"}, + {file = "websockets-13.1-cp311-cp311-win32.whl", hash = "sha256:c11d4d16e133f6df8916cc5b7e3e96ee4c44c936717d684a94f48f82edb7c92f"}, + {file = "websockets-13.1-cp311-cp311-win_amd64.whl", hash = "sha256:d04f13a1d75cb2b8382bdc16ae6fa58c97337253826dfe136195b7f89f661557"}, + {file = "websockets-13.1-py3-none-any.whl", hash = "sha256:a9a396a6ad26130cdae92ae10c36af09d9bfe6cafe69670fd3b6da9b07b4044f"}, + {file = "websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878"}, ] diff --git a/pyproject.toml b/pyproject.toml index aff08758..4fc77da6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "fastapi>=0.114.2,<1", "httpx>=0.27.2,<1", "jinja2>=3.1.4,<4", - "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.36.1", + "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@feature/mx-1702-type-prep", "neo4j>=5.24.0,<6", "pydantic>=2.9.1,<3", "starlette>=0.38.5,<1", diff --git a/tests/graph/test_query.py b/tests/graph/test_query.py index 80e27618..3b7a224d 100644 --- a/tests/graph/test_query.py +++ b/tests/graph/test_query.py @@ -111,7 +111,7 @@ def test_fetch_database_status(query_builder: QueryBuilder) -> None: } WITH n, collect(ref) as refs RETURN n{.*, entityType: head(labels(n)), _refs: refs} - ORDER BY n.identifier ASC + ORDER BY elementId(n) ASC SKIP $skip LIMIT $limit } @@ -150,7 +150,7 @@ def test_fetch_database_status(query_builder: QueryBuilder) -> None: } WITH n, collect(ref) as refs RETURN n{.*, entityType: head(labels(n)), _refs: refs} - ORDER BY n.identifier ASC + ORDER BY elementId(n) ASC SKIP $skip LIMIT $limit } @@ -222,7 +222,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, n.identifier ASC + ORDER BY merged.identifier, elementId(n) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip @@ -265,7 +265,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, n.identifier ASC + ORDER BY merged.identifier, elementId(n) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip diff --git a/tests/graph/test_transform.py b/tests/graph/test_transform.py index ae817f38..315ae191 100644 --- a/tests/graph/test_transform.py +++ b/tests/graph/test_transform.py @@ -1,7 +1,4 @@ -from pydantic import BaseModel - -from mex.backend.graph.transform import expand_references_in_search_result, to_primitive -from mex.common.types import APIType, MergedActivityIdentifier, YearMonth +from mex.backend.graph.transform import expand_references_in_search_result def test_expand_references_in_search_result() -> None: @@ -66,16 +63,3 @@ def test_expand_references_in_search_result() -> None: ], "title": [{"language": "de", "value": "Aktivität 1"}], } - - -def test_to_primitive() -> None: - class Test(BaseModel): - api: list[APIType] = [APIType["RPC"]] - activity: MergedActivityIdentifier = MergedActivityIdentifier.generate(seed=99) - month: YearMonth = YearMonth(2005, 11) - - assert to_primitive(Test()) == { - "api": ["https://mex.rki.de/item/api-type-5"], - "activity": "bFQoRhcVH5DHV1", - "month": "2005-11", - } From 614895185e0a53b6ff57e137b93e91fa2ad18e44 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Mon, 30 Sep 2024 16:49:37 +0200 Subject: [PATCH 02/10] clean up and CL --- CHANGELOG.md | 12 +++++++++++- mex/backend/ingest/main.py | 3 --- mex/backend/main.py | 9 ++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 635b1319..190c8dd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changes +- silence neo4j missing label warnings, because we will likely never need all labels +- sort search results by `elementId` instead of `identifier`, which is missing on rules, + this ensures a more stable sort order + ### Deprecated ### Removed +- remove already obsolete module `mex.backend.serialization` + this is not needed any more with the new mex-common version + ### Fixed +- fix how merged edges are counted (currently only used for debugging) + ### Security ## [0.19.1] - 2024-09-18 @@ -77,7 +86,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - remove stop-gaps for MX-1596 ### Removed -- removed class _BaseBulkIngestRequest for ingestion model + +- removed class _BaseBulkIngestRequest for ingestion model ## [0.17.0] - 2024-07-29 diff --git a/mex/backend/ingest/main.py b/mex/backend/ingest/main.py index 4130c565..3dca7a69 100644 --- a/mex/backend/ingest/main.py +++ b/mex/backend/ingest/main.py @@ -12,7 +12,4 @@ def ingest_extracted_items(request: BulkIngestRequest) -> BulkIngestResponse: """Ingest batches of extracted items grouped by their type.""" connector = GraphConnector.get() identifiers = connector.ingest(request.items) - from mex.common.logging import logger - - logger.info(request.model_dump_json(indent=2)) return BulkIngestResponse(identifiers=identifiers) diff --git a/mex/backend/main.py b/mex/backend/main.py index 1aa5f307..2b6b5c87 100644 --- a/mex/backend/main.py +++ b/mex/backend/main.py @@ -22,8 +22,11 @@ from mex.backend.settings import BackendSettings from mex.common.cli import entrypoint from mex.common.connector import CONNECTOR_STORE -from mex.common.types import EXTRACTED_IDENTIFIER_CLASSES, MERGED_IDENTIFIER_CLASSES -from mex.common.types import IDENTIFIER_PATTERN as MEX_ID_PATTERN +from mex.common.types import ( + EXTRACTED_IDENTIFIER_CLASSES, + IDENTIFIER_PATTERN, + MERGED_IDENTIFIER_CLASSES, +) def create_openapi_schema() -> dict[str, Any]: @@ -54,7 +57,7 @@ def create_openapi_schema() -> dict[str, Any]: "title": name, "type": "string", "description": identifier.__doc__, - "pattern": MEX_ID_PATTERN, + "pattern": IDENTIFIER_PATTERN, } app.openapi_schema = openapi_schema From d97c1d4c39d6145ba6988cb9f86b91846a3b8c34 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Mon, 30 Sep 2024 17:20:29 +0200 Subject: [PATCH 03/10] fix queries --- CHANGELOG.md | 3 +-- .../graph/cypher/fetch_extracted_or_rule_items.cql | 2 +- mex/backend/graph/cypher/fetch_merged_items.cql | 2 +- tests/graph/test_query.py | 8 ++++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 190c8dd5..2fe3d8a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changes - silence neo4j missing label warnings, because we will likely never need all labels -- sort search results by `elementId` instead of `identifier`, which is missing on rules, - this ensures a more stable sort order +- sort search results by `identifier` and `entityType` to ensure a more stable order ### Deprecated diff --git a/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql b/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql index 603cd4a3..cb13b436 100644 --- a/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql +++ b/mex/backend/graph/cypher/fetch_extracted_or_rule_items.cql @@ -59,7 +59,7 @@ CALL { } WITH n, collect(ref) as refs RETURN n{.*, entityType: head(labels(n)), _refs: refs} - ORDER BY elementId(n) ASC + ORDER BY n.identifier, n.entityType ASC SKIP $skip LIMIT $limit } diff --git a/mex/backend/graph/cypher/fetch_merged_items.cql b/mex/backend/graph/cypher/fetch_merged_items.cql index 56a26305..63bb2702 100644 --- a/mex/backend/graph/cypher/fetch_merged_items.cql +++ b/mex/backend/graph/cypher/fetch_merged_items.cql @@ -60,7 +60,7 @@ CALL { } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, elementId(n) ASC + ORDER BY merged.identifier ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip diff --git a/tests/graph/test_query.py b/tests/graph/test_query.py index 3b7a224d..fccb92df 100644 --- a/tests/graph/test_query.py +++ b/tests/graph/test_query.py @@ -111,7 +111,7 @@ def test_fetch_database_status(query_builder: QueryBuilder) -> None: } WITH n, collect(ref) as refs RETURN n{.*, entityType: head(labels(n)), _refs: refs} - ORDER BY elementId(n) ASC + ORDER BY n.identifier, n.entityType ASC SKIP $skip LIMIT $limit } @@ -150,7 +150,7 @@ def test_fetch_database_status(query_builder: QueryBuilder) -> None: } WITH n, collect(ref) as refs RETURN n{.*, entityType: head(labels(n)), _refs: refs} - ORDER BY elementId(n) ASC + ORDER BY n.identifier, n.entityType ASC SKIP $skip LIMIT $limit } @@ -222,7 +222,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, elementId(n) ASC + ORDER BY merged.identifier ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip @@ -265,7 +265,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, elementId(n) ASC + ORDER BY merged.identifier ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip From badf833469689cd206c630680611d4b5ef11f866 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Mon, 30 Sep 2024 17:40:59 +0200 Subject: [PATCH 04/10] fix tests --- .../graph/cypher/fetch_merged_items.cql | 2 +- tests/graph/test_connector.py | 30 +++++++++---------- tests/graph/test_query.py | 12 ++++---- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/mex/backend/graph/cypher/fetch_merged_items.cql b/mex/backend/graph/cypher/fetch_merged_items.cql index 63bb2702..7cb7b634 100644 --- a/mex/backend/graph/cypher/fetch_merged_items.cql +++ b/mex/backend/graph/cypher/fetch_merged_items.cql @@ -60,7 +60,7 @@ CALL { } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier ASC + ORDER BY merged.identifier, head(labels(n)) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip diff --git a/tests/graph/test_connector.py b/tests/graph/test_connector.py index f261891d..9b28479c 100644 --- a/tests/graph/test_connector.py +++ b/tests/graph/test_connector.py @@ -499,39 +499,39 @@ def test_fetch_merged_items() -> None: { "items": [ { + "identifier": "bFQoRhcVH5DHUB", "components": [ { + "entityType": "AdditiveOrganizationalUnit", "email": [], - "entityType": "ExtractedOrganizationalUnit", - "hadPrimarySource": ["bFQoRhcVH5DHUt"], - "identifier": "bFQoRhcVH5DHUA", - "identifierInPrimarySource": "ou-1.6", - "name": [{"language": "en", "value": "Unit 1.6"}], "parentUnit": ["bFQoRhcVH5DHUv"], "stableTargetId": ["bFQoRhcVH5DHUB"], + "name": [{"language": "en", "value": "Unit 1.7"}], + "website": [ + {"title": "Unit Homepage", "url": "https://unit-1-7"} + ], }, { - "entityType": "PreventiveOrganizationalUnit", + "identifierInPrimarySource": "ou-1.6", + "identifier": "bFQoRhcVH5DHUA", + "entityType": "ExtractedOrganizationalUnit", + "email": [], + "parentUnit": ["bFQoRhcVH5DHUv"], + "hadPrimarySource": ["bFQoRhcVH5DHUt"], "stableTargetId": ["bFQoRhcVH5DHUB"], + "name": [{"language": "en", "value": "Unit 1.6"}], }, { - "email": [], - "entityType": "SubtractiveOrganizationalUnit", + "entityType": "PreventiveOrganizationalUnit", "stableTargetId": ["bFQoRhcVH5DHUB"], }, { + "entityType": "SubtractiveOrganizationalUnit", "email": [], - "entityType": "AdditiveOrganizationalUnit", - "name": [{"language": "en", "value": "Unit 1.7"}], - "parentUnit": ["bFQoRhcVH5DHUv"], "stableTargetId": ["bFQoRhcVH5DHUB"], - "website": [ - {"title": "Unit Homepage", "url": "https://unit-1-7"} - ], }, ], "entityType": "MergedOrganizationalUnit", - "identifier": "bFQoRhcVH5DHUB", } ], "total": 8, diff --git a/tests/graph/test_query.py b/tests/graph/test_query.py index fccb92df..34fe8510 100644 --- a/tests/graph/test_query.py +++ b/tests/graph/test_query.py @@ -222,7 +222,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier ASC + ORDER BY merged.identifier, head(labels(n)) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip @@ -265,7 +265,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier ASC + ORDER BY merged.identifier, head(labels(n)) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip @@ -392,7 +392,7 @@ def test_fetch_identities( MERGE (source)-[edge:agendaSignedOff {position: $ref_positions[2]}]->(target_2) RETURN edge } -WITH source, collect(edge) as edges +WITH source, count(edge) as merged, collect(edge) as edges CALL { WITH source, edges MATCH (source)-[outdated_edge]->(:MergedThis|MergedThat|MergedOther) @@ -400,7 +400,7 @@ def test_fetch_identities( DELETE outdated_edge RETURN count(outdated_edge) as pruned } -RETURN count(edges) as merged, pruned, edges;""", +RETURN merged, pruned, edges;""", ), ( [], @@ -409,7 +409,7 @@ def test_fetch_identities( CALL { RETURN null as edge } -WITH source, collect(edge) as edges +WITH source, count(edge) as merged, collect(edge) as edges CALL { WITH source, edges MATCH (source)-[outdated_edge]->(:MergedThis|MergedThat|MergedOther) @@ -417,7 +417,7 @@ def test_fetch_identities( DELETE outdated_edge RETURN count(outdated_edge) as pruned } -RETURN count(edges) as merged, pruned, edges;""", +RETURN merged, pruned, edges;""", ), ], ids=["has-ref-labels", "no-ref-labels"], From 01225f04b1893ff21b9932323f7b62368dd2cc58 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Thu, 10 Oct 2024 17:18:01 +0200 Subject: [PATCH 05/10] handle errors better --- mex/backend/exceptions.py | 52 +++++++++++++++++----------- mex/backend/main.py | 5 +-- tests/test_exceptions.py | 71 ++++++++++++++++++++------------------- 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/mex/backend/exceptions.py b/mex/backend/exceptions.py index 50647ca3..e2b0d13c 100644 --- a/mex/backend/exceptions.py +++ b/mex/backend/exceptions.py @@ -1,10 +1,10 @@ -from typing import Any +from typing import Any, cast from fastapi.encoders import jsonable_encoder -from fastapi.responses import JSONResponse from pydantic import BaseModel, ValidationError from starlette import status from starlette.requests import Request +from starlette.responses import Response from mex.common.logging import logger @@ -34,23 +34,35 @@ class ErrorResponse(BaseModel): debug: DebuggingInfo -def handle_uncaught_exception(request: Request, exc: Exception) -> JSONResponse: +def handle_validation_error(request: Request, exc: Exception) -> Response: + """Handle pydantic validation errors and provide debugging info.""" + logger.exception("ValidationError %s", exc) + return Response( + content=ErrorResponse( + message=str(exc), + debug=DebuggingInfo( + errors=[ + jsonable_encoder(e) for e in cast(ValidationError, exc).errors() + ], + scope=DebuggingScope.model_validate(request.scope), + ), + ).model_dump_json(), + status_code=status.HTTP_400_BAD_REQUEST, + media_type="application/json", + ) + + +def handle_uncaught_exception(request: Request, exc: Exception) -> Response: """Handle uncaught errors and provide debugging info.""" - logger.exception("Error %s", exc) - if isinstance(exc, ValidationError): - errors = [dict(error) for error in exc.errors()] - status_code = status.HTTP_400_BAD_REQUEST - else: - errors = [dict(type=type(exc).__name__)] - status_code = status.HTTP_500_INTERNAL_SERVER_ERROR - return JSONResponse( - jsonable_encoder( - ErrorResponse( - message=str(exc), - debug=DebuggingInfo( - errors=errors, scope=DebuggingScope.model_validate(request.scope) - ), - ) - ), - status_code, + logger.exception("UncaughtError %s", exc) + return Response( + content=ErrorResponse( + message=str(exc), + debug=DebuggingInfo( + errors=[dict(type=type(exc).__name__)], + scope=DebuggingScope.model_validate(request.scope), + ), + ).model_dump_json(), + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + media_type="application/json", ) diff --git a/mex/backend/main.py b/mex/backend/main.py index 2b6b5c87..fd0d565d 100644 --- a/mex/backend/main.py +++ b/mex/backend/main.py @@ -7,10 +7,10 @@ from fastapi import APIRouter, Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.utils import get_openapi -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError from mex.backend.auxiliary.wikidata import router as wikidata_router -from mex.backend.exceptions import handle_uncaught_exception +from mex.backend.exceptions import handle_uncaught_exception, handle_validation_error from mex.backend.extracted.main import router as extracted_router from mex.backend.identity.main import router as identity_router from mex.backend.ingest.main import router as ingest_router @@ -110,6 +110,7 @@ def check_system_status() -> SystemStatus: app.include_router(router) +app.add_exception_handler(ValidationError, handle_validation_error) app.add_exception_handler(Exception, handle_uncaught_exception) app.add_middleware( CORSMiddleware, diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 499650d8..0a605683 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -6,7 +6,7 @@ import pytest from pydantic import ValidationError -from mex.backend.exceptions import handle_uncaught_exception +from mex.backend.exceptions import handle_uncaught_exception, handle_validation_error from mex.common.exceptions import MExError MOCK_REQUEST_SCOPE = { @@ -33,38 +33,6 @@ }, 500, ), - ( - ValidationError.from_exception_data( - "foo", - [ - { - "type": pydantic_core.PydanticCustomError( - "TestError", "You messed up!" - ), - "loc": ("integerAttribute",), - "input": "mumbo-jumbo", - } - ], - ), - { - "debug": { - "errors": [ - { - "input": "mumbo-jumbo", - "loc": ["integerAttribute"], - "msg": "You messed up!", - "type": "TestError", - } - ], - "scope": MOCK_REQUEST_SCOPE, - }, - "message": "1 validation error for foo\n" - "integerAttribute\n" - " You messed up! [type=TestError, input_value='mumbo-jumbo', " - "input_type=str]", - }, - 400, - ), ( MExError("bar"), { @@ -77,7 +45,7 @@ 500, ), ], - ids=["TypeError", "ValidationError", "MExError"], + ids=["TypeError", "MExError"], ) def test_handle_uncaught_exception( exception: Exception, expected: dict[str, Any], status_code: int @@ -86,3 +54,38 @@ def test_handle_uncaught_exception( response = handle_uncaught_exception(request, exception) assert response.status_code == status_code, response.body assert json.loads(response.body) == expected + + +def test_handle_validation_error() -> None: + request = Mock(scope=MOCK_REQUEST_SCOPE) + exception = ValidationError.from_exception_data( + "foo", + [ + { + "type": pydantic_core.PydanticCustomError( + "TestError", "You messed up!" + ), + "loc": ("integerAttribute",), + "input": "mumbo-jumbo", + } + ], + ) + response = handle_validation_error(request, exception) + assert response.status_code == 400, response.body + assert json.loads(response.body) == { + "debug": { + "errors": [ + { + "input": "mumbo-jumbo", + "loc": ["integerAttribute"], + "msg": "You messed up!", + "type": "TestError", + } + ], + "scope": MOCK_REQUEST_SCOPE, + }, + "message": "1 validation error for foo\n" + "integerAttribute\n" + " You messed up! [type=TestError, input_value='mumbo-jumbo', " + "input_type=str]", + } From b0185667cf801cd26958f0ca0e785cac8f7558de Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Fri, 11 Oct 2024 15:11:08 +0200 Subject: [PATCH 06/10] update lock --- pdm.lock | 172 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 87 insertions(+), 87 deletions(-) diff --git a/pdm.lock b/pdm.lock index b4a03869..96dece05 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:5695fbba8be6dfc6f25fd616d36b3ea88aaff16159ba7269baed5730451fd0cf" +content_hash = "sha256:78e8d090e4daf2fb2387d3bd84d4a5d15e270160910dc670ff0ec75e0f2d466f" [[metadata.targets]] requires_python = "==3.11.*" @@ -99,8 +99,8 @@ files = [ [[package]] name = "black" -version = "24.8.0" -requires_python = ">=3.8" +version = "24.10.0" +requires_python = ">=3.9" summary = "The uncompromising code formatter." groups = ["dev"] marker = "python_version == \"3.11\"" @@ -114,12 +114,12 @@ dependencies = [ "typing-extensions>=4.0.1; python_version < \"3.11\"", ] files = [ - {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, - {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, - {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, - {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, - {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, - {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, + {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, + {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, + {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, + {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, + {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, + {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, ] [[package]] @@ -136,29 +136,29 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.3.2" +version = "3.4.0" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." groups = ["default", "dev"] marker = "python_version == \"3.11\"" files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, ] [[package]] @@ -191,49 +191,49 @@ files = [ [[package]] name = "coverage" -version = "7.6.1" -requires_python = ">=3.8" +version = "7.6.2" +requires_python = ">=3.9" summary = "Code coverage measurement for Python" groups = ["dev"] marker = "python_version == \"3.11\"" files = [ - {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, - {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, - {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, - {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, - {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, + {file = "coverage-7.6.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bb21bac7783c1bf6f4bbe68b1e0ff0d20e7e7732cfb7995bc8d96e23aa90fc7b"}, + {file = "coverage-7.6.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b2e437fbd8fae5bc7716b9c7ff97aecc95f0b4d56e4ca08b3c8d8adcaadb84"}, + {file = "coverage-7.6.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:536f77f2bf5797983652d1d55f1a7272a29afcc89e3ae51caa99b2db4e89d658"}, + {file = "coverage-7.6.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f361296ca7054f0936b02525646b2731b32c8074ba6defab524b79b2b7eeac72"}, + {file = "coverage-7.6.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7926d8d034e06b479797c199747dd774d5e86179f2ce44294423327a88d66ca7"}, + {file = "coverage-7.6.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0bbae11c138585c89fb4e991faefb174a80112e1a7557d507aaa07675c62e66b"}, + {file = "coverage-7.6.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fcad7d5d2bbfeae1026b395036a8aa5abf67e8038ae7e6a25c7d0f88b10a8e6a"}, + {file = "coverage-7.6.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f01e53575f27097d75d42de33b1b289c74b16891ce576d767ad8c48d17aeb5e0"}, + {file = "coverage-7.6.2-cp311-cp311-win32.whl", hash = "sha256:7781f4f70c9b0b39e1b129b10c7d43a4e0c91f90c60435e6da8288efc2b73438"}, + {file = "coverage-7.6.2-cp311-cp311-win_amd64.whl", hash = "sha256:9bcd51eeca35a80e76dc5794a9dd7cb04b97f0e8af620d54711793bfc1fbba4b"}, + {file = "coverage-7.6.2.tar.gz", hash = "sha256:a5f81e68aa62bc0cfca04f7b19eaa8f9c826b53fc82ab9e2121976dc74f131f3"}, ] [[package]] name = "coverage" -version = "7.6.1" +version = "7.6.2" extras = ["toml"] -requires_python = ">=3.8" +requires_python = ">=3.9" summary = "Code coverage measurement for Python" groups = ["dev"] marker = "python_version == \"3.11\"" dependencies = [ - "coverage==7.6.1", + "coverage==7.6.2", "tomli; python_full_version <= \"3.11.0a6\"", ] files = [ - {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, - {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, - {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, - {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, - {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, + {file = "coverage-7.6.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bb21bac7783c1bf6f4bbe68b1e0ff0d20e7e7732cfb7995bc8d96e23aa90fc7b"}, + {file = "coverage-7.6.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b2e437fbd8fae5bc7716b9c7ff97aecc95f0b4d56e4ca08b3c8d8adcaadb84"}, + {file = "coverage-7.6.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:536f77f2bf5797983652d1d55f1a7272a29afcc89e3ae51caa99b2db4e89d658"}, + {file = "coverage-7.6.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f361296ca7054f0936b02525646b2731b32c8074ba6defab524b79b2b7eeac72"}, + {file = "coverage-7.6.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7926d8d034e06b479797c199747dd774d5e86179f2ce44294423327a88d66ca7"}, + {file = "coverage-7.6.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0bbae11c138585c89fb4e991faefb174a80112e1a7557d507aaa07675c62e66b"}, + {file = "coverage-7.6.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fcad7d5d2bbfeae1026b395036a8aa5abf67e8038ae7e6a25c7d0f88b10a8e6a"}, + {file = "coverage-7.6.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f01e53575f27097d75d42de33b1b289c74b16891ce576d767ad8c48d17aeb5e0"}, + {file = "coverage-7.6.2-cp311-cp311-win32.whl", hash = "sha256:7781f4f70c9b0b39e1b129b10c7d43a4e0c91f90c60435e6da8288efc2b73438"}, + {file = "coverage-7.6.2-cp311-cp311-win_amd64.whl", hash = "sha256:9bcd51eeca35a80e76dc5794a9dd7cb04b97f0e8af620d54711793bfc1fbba4b"}, + {file = "coverage-7.6.2.tar.gz", hash = "sha256:a5f81e68aa62bc0cfca04f7b19eaa8f9c826b53fc82ab9e2121976dc74f131f3"}, ] [[package]] @@ -509,23 +509,23 @@ files = [ [[package]] name = "markupsafe" -version = "2.1.5" -requires_python = ">=3.7" +version = "3.0.1" +requires_python = ">=3.9" summary = "Safely add untrusted strings to HTML/XML markup." groups = ["default", "dev"] marker = "python_version == \"3.11\"" files = [ - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, + {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, ] [[package]] @@ -545,11 +545,11 @@ files = [ [[package]] name = "mex-common" -version = "0.36.1" -requires_python = "<3.13,>=3.11" +version = "0.38.0" +requires_python = ">=3.11,<3.13" git = "https://github.com/robert-koch-institut/mex-common.git" -ref = "feature/mx-1702-type-prep" -revision = "612c7cc4471eefd9b7ed1d89b7ff49f52d3a46d0" +ref = "0.38.0" +revision = "c428ac48a1b6ff458573ea35374cefc7ef6cf025" summary = "Common library for MEx python projects." groups = ["default"] marker = "python_version == \"3.11\"" @@ -1083,7 +1083,7 @@ files = [ [[package]] name = "sphinx" -version = "8.0.2" +version = "8.1.0" requires_python = ">=3.10" summary = "Python documentation generator" groups = ["dev"] @@ -1099,17 +1099,17 @@ dependencies = [ "packaging>=23.0", "requests>=2.30.0", "snowballstemmer>=2.2", - "sphinxcontrib-applehelp", - "sphinxcontrib-devhelp", - "sphinxcontrib-htmlhelp>=2.0.0", - "sphinxcontrib-jsmath", - "sphinxcontrib-qthelp", + "sphinxcontrib-applehelp>=1.0.7", + "sphinxcontrib-devhelp>=1.0.6", + "sphinxcontrib-htmlhelp>=2.0.6", + "sphinxcontrib-jsmath>=1.0.1", + "sphinxcontrib-qthelp>=1.0.6", "sphinxcontrib-serializinghtml>=1.1.9", "tomli>=2; python_version < \"3.11\"", ] files = [ - {file = "sphinx-8.0.2-py3-none-any.whl", hash = "sha256:56173572ae6c1b9a38911786e206a110c9749116745873feae4f9ce88e59391d"}, - {file = "sphinx-8.0.2.tar.gz", hash = "sha256:0cce1ddcc4fd3532cf1dd283bc7d886758362c5c1de6598696579ce96d8ffa5b"}, + {file = "sphinx-8.1.0-py3-none-any.whl", hash = "sha256:3202bba95697b9fc4371a07d6d457239de9860244ce235283149f817c253fd2f"}, + {file = "sphinx-8.1.0.tar.gz", hash = "sha256:109454425dbf4c78ecfdd481e56f078376d077edbda29804dba05c5161c8de06"}, ] [[package]] @@ -1278,7 +1278,7 @@ files = [ [[package]] name = "uvicorn" -version = "0.31.0" +version = "0.31.1" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["default"] @@ -1289,13 +1289,13 @@ dependencies = [ "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ - {file = "uvicorn-0.31.0-py3-none-any.whl", hash = "sha256:cac7be4dd4d891c363cd942160a7b02e69150dcbc7a36be04d5f4af4b17c8ced"}, - {file = "uvicorn-0.31.0.tar.gz", hash = "sha256:13bc21373d103859f68fe739608e2eb054a816dea79189bc3ca08ea89a275906"}, + {file = "uvicorn-0.31.1-py3-none-any.whl", hash = "sha256:adc42d9cac80cf3e51af97c1851648066841e7cfb6993a4ca8de29ac1548ed41"}, + {file = "uvicorn-0.31.1.tar.gz", hash = "sha256:f5167919867b161b7bcaf32646c6a94cdbd4c3aa2eb5c17d36bb9aa5cfd8c493"}, ] [[package]] name = "uvicorn" -version = "0.31.0" +version = "0.31.1" extras = ["standard"] requires_python = ">=3.8" summary = "The lightning-fast ASGI server." @@ -1306,14 +1306,14 @@ dependencies = [ "httptools>=0.5.0", "python-dotenv>=0.13", "pyyaml>=5.1", - "uvicorn==0.31.0", + "uvicorn==0.31.1", "uvloop!=0.15.0,!=0.15.1,>=0.14.0; (sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"", "watchfiles>=0.13", "websockets>=10.4", ] files = [ - {file = "uvicorn-0.31.0-py3-none-any.whl", hash = "sha256:cac7be4dd4d891c363cd942160a7b02e69150dcbc7a36be04d5f4af4b17c8ced"}, - {file = "uvicorn-0.31.0.tar.gz", hash = "sha256:13bc21373d103859f68fe739608e2eb054a816dea79189bc3ca08ea89a275906"}, + {file = "uvicorn-0.31.1-py3-none-any.whl", hash = "sha256:adc42d9cac80cf3e51af97c1851648066841e7cfb6993a4ca8de29ac1548ed41"}, + {file = "uvicorn-0.31.1.tar.gz", hash = "sha256:f5167919867b161b7bcaf32646c6a94cdbd4c3aa2eb5c17d36bb9aa5cfd8c493"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 1cbbfc91..1aa6d941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "fastapi>=0.114.2,<1", "httpx>=0.27.2,<1", "jinja2>=3.1.4,<4", - "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@feature/mx-1702-type-prep", + "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.38.0", "neo4j>=5.24.0,<6", "pydantic>=2.9.1,<3", "starlette>=0.38.5,<1", From 099223206da92a8d0e88b3ad98e9cc6709226226 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Fri, 11 Oct 2024 15:13:22 +0200 Subject: [PATCH 07/10] CL --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fe3d8a5..af9220a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - silence neo4j missing label warnings, because we will likely never need all labels - sort search results by `identifier` and `entityType` to ensure a more stable order +- improve handling of pydantic validation errors and uncaught errors ### Deprecated From 3dee885a4fd6dfb24fae27e38c3ce99b7d337bd2 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Wed, 16 Oct 2024 13:29:41 +0200 Subject: [PATCH 08/10] dont blame graph validation errors on the user --- mex/backend/extracted/helpers.py | 18 +++++++++++++-- mex/backend/graph/exceptions.py | 4 ++++ mex/backend/ingest/helpers.py | 31 ++++++++++++++++++++++++++ mex/backend/ingest/main.py | 8 +++---- mex/backend/merged/helpers.py | 31 ++++++++++++++++++++------ mex/backend/utils.py | 38 +++++++++++++++++++++++++++++++- tests/test_utils.py | 34 +++++++++++++++++++++++++++- 7 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 mex/backend/ingest/helpers.py diff --git a/mex/backend/extracted/helpers.py b/mex/backend/extracted/helpers.py index b521451b..7244394a 100644 --- a/mex/backend/extracted/helpers.py +++ b/mex/backend/extracted/helpers.py @@ -1,5 +1,9 @@ +from pydantic import ValidationError + from mex.backend.extracted.models import ExtractedItemSearch from mex.backend.graph.connector import GraphConnector +from mex.backend.graph.exceptions import InconsistentGraphError +from mex.backend.utils import reraising from mex.common.models import AnyExtractedModel @@ -19,6 +23,9 @@ def search_extracted_items_in_graph( skip: How many items to skip for pagination limit: How many items to return at most + Raises: + InconsistentGraphError: When the graph response cannot be parsed + Returns: ExtractedItemSearch instance """ @@ -30,8 +37,12 @@ def search_extracted_items_in_graph( skip=skip, limit=limit, ) - search_result = graph_result.one() - return ExtractedItemSearch.model_validate(search_result) + return reraising( + ValidationError, + InconsistentGraphError, + ExtractedItemSearch.model_validate, + graph_result.one(), + ) def get_extracted_items_from_graph( @@ -46,6 +57,9 @@ def get_extracted_items_from_graph( entity_type: Optional entity type filter limit: How many items to return at most + Raises: + InconsistentGraphError: When the graph response cannot be parsed + Returns: List of extracted items """ diff --git a/mex/backend/graph/exceptions.py b/mex/backend/graph/exceptions.py index bba541d3..fad097f8 100644 --- a/mex/backend/graph/exceptions.py +++ b/mex/backend/graph/exceptions.py @@ -7,3 +7,7 @@ class NoResultFoundError(MExError): class MultipleResultsFoundError(MExError): """A single database result was required but more than one were found.""" + + +class InconsistentGraphError(MExError): + """Exception raised for inconsistencies found in the graph database.""" diff --git a/mex/backend/ingest/helpers.py b/mex/backend/ingest/helpers.py new file mode 100644 index 00000000..46dff70b --- /dev/null +++ b/mex/backend/ingest/helpers.py @@ -0,0 +1,31 @@ +from pydantic import ValidationError + +from mex.backend.graph.connector import GraphConnector +from mex.backend.graph.exceptions import InconsistentGraphError +from mex.backend.ingest.models import BulkIngestResponse +from mex.backend.utils import reraising +from mex.common.models import AnyExtractedModel + + +def ingest_extracted_items_into_graph( + items: list[AnyExtractedModel], +) -> BulkIngestResponse: + """Ingest a batch of extracted items and return their identifiers. + + Args: + items: list of AnyExtractedModel + + Raises: + InconsistentGraphError: When the graph response cannot be parsed + + Returns: + List of identifiers of the ingested items + """ + connector = GraphConnector.get() + identifiers = connector.ingest(items) + return reraising( + ValidationError, + InconsistentGraphError, + BulkIngestResponse, + identifiers=identifiers, + ) diff --git a/mex/backend/ingest/main.py b/mex/backend/ingest/main.py index 3dca7a69..67963eb8 100644 --- a/mex/backend/ingest/main.py +++ b/mex/backend/ingest/main.py @@ -1,7 +1,7 @@ from fastapi import APIRouter from starlette import status -from mex.backend.graph.connector import GraphConnector +from mex.backend.ingest.helpers import ingest_extracted_items_into_graph from mex.backend.ingest.models import BulkIngestRequest, BulkIngestResponse router = APIRouter() @@ -9,7 +9,5 @@ @router.post("/ingest", status_code=status.HTTP_201_CREATED, tags=["extractors"]) def ingest_extracted_items(request: BulkIngestRequest) -> BulkIngestResponse: - """Ingest batches of extracted items grouped by their type.""" - connector = GraphConnector.get() - identifiers = connector.ingest(request.items) - return BulkIngestResponse(identifiers=identifiers) + """Ingest a batch of extracted items and return their identifiers.""" + return ingest_extracted_items_into_graph(request.items) diff --git a/mex/backend/merged/helpers.py b/mex/backend/merged/helpers.py index 45a8fce4..aa863391 100644 --- a/mex/backend/merged/helpers.py +++ b/mex/backend/merged/helpers.py @@ -1,12 +1,13 @@ -from typing import Annotated, Any +from typing import Annotated, Any, cast -from pydantic import Field, TypeAdapter +from pydantic import Field, TypeAdapter, ValidationError from mex.backend.fields import MERGEABLE_FIELDS_BY_CLASS_NAME from mex.backend.graph.connector import GraphConnector +from mex.backend.graph.exceptions import InconsistentGraphError from mex.backend.merged.models import MergedItemSearch from mex.backend.rules.helpers import transform_raw_rules_to_rule_set_response -from mex.backend.utils import extend_list_in_dict, prune_list_in_dict +from mex.backend.utils import extend_list_in_dict, prune_list_in_dict, reraising from mex.common.exceptions import MExError from mex.common.models import ( EXTRACTED_MODEL_CLASSES_BY_NAME, @@ -97,6 +98,9 @@ def create_merged_item( extracted_items: List of extracted items, can be empty rule_set: Rule set, with potentially empty rules + Raises: + InconsistentGraphError: When the graph response cannot be parsed + Returns: Instance of a merged item """ @@ -117,8 +121,13 @@ def create_merged_item( if rule_set: _apply_additive_rule(merged_dict, fields, rule_set.additive) _apply_subtractive_rule(merged_dict, fields, rule_set.subtractive) - - return cls.model_validate(merged_dict) + merged_item = reraising( + ValidationError, + InconsistentGraphError, + cls.model_validate, + merged_dict, + ) + return cast(AnyMergedModel, merged_item) # mypy, get a grip! def search_merged_items_in_graph( @@ -137,6 +146,9 @@ def search_merged_items_in_graph( skip: How many items to skip for pagination limit: How many items to return at most + Raises: + InconsistentGraphError: When the graph response cannot be parsed + Returns: MergedItemSearch instance """ @@ -182,5 +194,10 @@ def search_merged_items_in_graph( rule_set=rule_set_response, ) ) - - return MergedItemSearch(items=items, total=total) + return reraising( + ValidationError, + InconsistentGraphError, + MergedItemSearch, + items=items, + total=total, + ) diff --git a/mex/backend/utils.py b/mex/backend/utils.py index b6e1c769..ec883867 100644 --- a/mex/backend/utils.py +++ b/mex/backend/utils.py @@ -1,6 +1,8 @@ -from typing import TypeVar +from collections.abc import Callable +from typing import ParamSpec, TypeVar T = TypeVar("T") +P = ParamSpec("P") def extend_list_in_dict(dict_: dict[str, list[T]], key: str, item: list[T] | T) -> None: @@ -23,3 +25,37 @@ def prune_list_in_dict(dict_: dict[str, list[T]], key: str, item: list[T] | T) - list_.remove(removable) except ValueError: pass + + +def reraising( + original_error: type[Exception], + reraise_as: type[Exception], + fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, +) -> T: + """Execute a function and catch a specific exception, re-raising it as a different. + + This utility allows you to call any given function `fn` with its arguments `args` + and `kwargs`, catch a specified exception type (`original_error`), and re-raise + it as another exception type (`reraise_as`) while preserving the original traceback. + + Args: + original_error: The exception class to catch. + reraise_as: The exception class to re-raise as. + fn: The function to be called. + *args: Positional arguments to be passed to the function `fn`. + **kwargs: Keyword arguments to be passed to the function `fn`. + + Raises: + reraise_as: If `fn` raises an exception of type `original_error`, it is caught + and re-raised as `reraise_as`, while preserving the original + exception as the cause. + + Returns: + The return value of the function `fn` if it executes successfully. + """ + try: + return fn(*args, **kwargs) + except original_error as error: + raise reraise_as from error diff --git a/tests/test_utils.py b/tests/test_utils.py index 6543f4a0..858bc37e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,6 @@ -from mex.backend.utils import extend_list_in_dict, prune_list_in_dict +import pytest + +from mex.backend.utils import extend_list_in_dict, prune_list_in_dict, reraising def test_extend_list_in_dict() -> None: @@ -35,3 +37,33 @@ def test_prune_list_in_dict() -> None: "prune-me": ["42"], "does-not-exist": [], } + + +def test_reraising_no_exception() -> None: + def add(a: int, b: int) -> int: + return a + b + + result = reraising(ValueError, RuntimeError, add, 1, 2) + assert result == 3 + + +def test_reraising_with_caught_exception() -> None: + def divide(a: int, b: int) -> float: + return a / b + + with pytest.raises(ValueError) as exc_info: + reraising(ZeroDivisionError, ValueError, divide, 1, 0) + + assert isinstance(exc_info.value, ValueError) + assert exc_info.value.__cause__ is not None + assert isinstance(exc_info.value.__cause__, ZeroDivisionError) + + +def test_reraising_propagates_other_exceptions() -> None: + def raise_type_error() -> None: + raise TypeError("This is a TypeError") + + with pytest.raises(TypeError) as exc_info: + reraising(ValueError, RuntimeError, raise_type_error) + + assert isinstance(exc_info.value, TypeError) From caa41cb453256478c3698eae55727a3d6197e3a4 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Wed, 16 Oct 2024 14:49:51 +0200 Subject: [PATCH 09/10] sort by id *and* label --- mex/backend/graph/cypher/fetch_merged_items.cql | 2 +- tests/graph/test_query.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mex/backend/graph/cypher/fetch_merged_items.cql b/mex/backend/graph/cypher/fetch_merged_items.cql index 7cb7b634..a96447bd 100644 --- a/mex/backend/graph/cypher/fetch_merged_items.cql +++ b/mex/backend/graph/cypher/fetch_merged_items.cql @@ -60,7 +60,7 @@ CALL { } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, head(labels(n)) ASC + ORDER BY merged.identifier, n.identifier, head(labels(n)) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip diff --git a/tests/graph/test_query.py b/tests/graph/test_query.py index 34fe8510..c0f21cd0 100644 --- a/tests/graph/test_query.py +++ b/tests/graph/test_query.py @@ -222,7 +222,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, head(labels(n)) ASC + ORDER BY merged.identifier, n.identifier, head(labels(n)) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip @@ -265,7 +265,7 @@ def test_fetch_extracted_items( } ELSE NULL END as ref } WITH merged, n, collect(ref) as refs - ORDER BY merged.identifier, head(labels(n)) ASC + ORDER BY merged.identifier, n.identifier, head(labels(n)) ASC WITH merged, collect(n{.*, entityType: head(labels(n)), _refs: refs}) as n RETURN merged{entityType: head(labels(merged)), identifier: merged.identifier, components: n} SKIP $skip From dbcddca524bd5a4bdd3a42fda0259a3fe3d4ed48 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Wed, 16 Oct 2024 14:56:11 +0200 Subject: [PATCH 10/10] fix order --- tests/graph/test_connector.py | 142 ++++++++++++++++------------------ 1 file changed, 68 insertions(+), 74 deletions(-) diff --git a/tests/graph/test_connector.py b/tests/graph/test_connector.py index 9b28479c..a6f5bc2a 100644 --- a/tests/graph/test_connector.py +++ b/tests/graph/test_connector.py @@ -244,20 +244,18 @@ def test_fetch_extracted_items() -> None: result = connector.fetch_extracted_items(None, None, None, 0, 1) - assert result.all() == [ - { - "items": [ - { - "entityType": MEX_EXTRACTED_PRIMARY_SOURCE.entityType, - "hadPrimarySource": [MEX_EXTRACTED_PRIMARY_SOURCE.hadPrimarySource], - "identifier": MEX_EXTRACTED_PRIMARY_SOURCE.identifier, - "identifierInPrimarySource": MEX_EXTRACTED_PRIMARY_SOURCE.identifierInPrimarySource, - "stableTargetId": [MEX_EXTRACTED_PRIMARY_SOURCE.stableTargetId], - } - ], - "total": 8, - } - ] + assert result.one() == { + "items": [ + { + "entityType": MEX_EXTRACTED_PRIMARY_SOURCE.entityType, + "hadPrimarySource": [MEX_EXTRACTED_PRIMARY_SOURCE.hadPrimarySource], + "identifier": MEX_EXTRACTED_PRIMARY_SOURCE.identifier, + "identifierInPrimarySource": MEX_EXTRACTED_PRIMARY_SOURCE.identifierInPrimarySource, + "stableTargetId": [MEX_EXTRACTED_PRIMARY_SOURCE.stableTargetId], + } + ], + "total": 8, + } @pytest.mark.integration @@ -266,7 +264,7 @@ def test_fetch_extracted_items_empty() -> None: result = connector.fetch_extracted_items(None, "thisIdDoesNotExist", None, 0, 1) - assert result.all() == [{"items": [], "total": 0}] + assert result.one() == {"items": [], "total": 0} @pytest.mark.usefixtures("mocked_query_builder") @@ -364,21 +362,19 @@ def test_fetch_rule_items( result = connector.fetch_rule_items(None, None, None, 0, 1) - assert result.all() == [ - { - "items": [ - { - "email": [], - "entityType": "AdditiveOrganizationalUnit", - "name": [dict(value="Unit 1.7", language="en")], - "website": [dict(title="Unit Homepage", url="https://unit-1-7")], - "parentUnit": [load_dummy_rule_set.additive.parentUnit], - "stableTargetId": ["bFQoRhcVH5DHUB"], - } - ], - "total": 3, - } - ] + assert result.one() == { + "items": [ + { + "email": [], + "entityType": "AdditiveOrganizationalUnit", + "name": [dict(value="Unit 1.7", language="en")], + "website": [dict(title="Unit Homepage", url="https://unit-1-7")], + "parentUnit": [load_dummy_rule_set.additive.parentUnit], + "stableTargetId": ["bFQoRhcVH5DHUB"], + } + ], + "total": 3, + } @pytest.mark.integration @@ -387,7 +383,7 @@ def test_fetch_rule_items_empty() -> None: result = connector.fetch_rule_items(None, "thisIdDoesNotExist", None, 0, 1) - assert result.all() == [{"items": [], "total": 0}] + assert result.one() == {"items": [], "total": 0} @pytest.mark.usefixtures("mocked_query_builder") @@ -495,48 +491,46 @@ def test_fetch_merged_items() -> None: result = connector.fetch_merged_items(None, None, None, 1, 1) - assert result.all() == [ - { - "items": [ - { - "identifier": "bFQoRhcVH5DHUB", - "components": [ - { - "entityType": "AdditiveOrganizationalUnit", - "email": [], - "parentUnit": ["bFQoRhcVH5DHUv"], - "stableTargetId": ["bFQoRhcVH5DHUB"], - "name": [{"language": "en", "value": "Unit 1.7"}], - "website": [ - {"title": "Unit Homepage", "url": "https://unit-1-7"} - ], - }, - { - "identifierInPrimarySource": "ou-1.6", - "identifier": "bFQoRhcVH5DHUA", - "entityType": "ExtractedOrganizationalUnit", - "email": [], - "parentUnit": ["bFQoRhcVH5DHUv"], - "hadPrimarySource": ["bFQoRhcVH5DHUt"], - "stableTargetId": ["bFQoRhcVH5DHUB"], - "name": [{"language": "en", "value": "Unit 1.6"}], - }, - { - "entityType": "PreventiveOrganizationalUnit", - "stableTargetId": ["bFQoRhcVH5DHUB"], - }, - { - "entityType": "SubtractiveOrganizationalUnit", - "email": [], - "stableTargetId": ["bFQoRhcVH5DHUB"], - }, - ], - "entityType": "MergedOrganizationalUnit", - } - ], - "total": 8, - } - ] + assert result.one() == { + "items": [ + { + "components": [ + { + "email": [], + "entityType": "ExtractedOrganizationalUnit", + "hadPrimarySource": ["bFQoRhcVH5DHUt"], + "identifier": "bFQoRhcVH5DHUA", + "identifierInPrimarySource": "ou-1.6", + "name": [{"language": "en", "value": "Unit 1.6"}], + "parentUnit": ["bFQoRhcVH5DHUv"], + "stableTargetId": ["bFQoRhcVH5DHUB"], + }, + { + "email": [], + "entityType": "AdditiveOrganizationalUnit", + "name": [{"language": "en", "value": "Unit 1.7"}], + "parentUnit": ["bFQoRhcVH5DHUv"], + "stableTargetId": ["bFQoRhcVH5DHUB"], + "website": [ + {"title": "Unit Homepage", "url": "https://unit-1-7"} + ], + }, + { + "entityType": "PreventiveOrganizationalUnit", + "stableTargetId": ["bFQoRhcVH5DHUB"], + }, + { + "email": [], + "entityType": "SubtractiveOrganizationalUnit", + "stableTargetId": ["bFQoRhcVH5DHUB"], + }, + ], + "entityType": "MergedOrganizationalUnit", + "identifier": "bFQoRhcVH5DHUB", + } + ], + "total": 8, + } @pytest.mark.integration @@ -545,7 +539,7 @@ def test_fetch_merged_items_empty() -> None: result = connector.fetch_merged_items(None, "thisIdDoesNotExist", None, 0, 1) - assert result.all() == [{"items": [], "total": 0}] + assert result.one() == {"items": [], "total": 0} @pytest.mark.usefixtures("mocked_query_builder")