feature/mx-1381 rework database model

.pre-commit-config.yaml

@@ -3,12 +3,12 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.3.5
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
   - repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.3.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/pre-commit-hooks

CHANGELOG.md

@@ -11,10 +11,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changes
 
+- re-implemented queries as templated cql files
+- updated graph connector for new queries
+- improved isolation of neo4j dependency
+- improved documentation and code-readability
+
 ### Deprecated
 
 ### Removed
 
+- trashed hydration module
+
 ### Fixed
 
 ### Security

mex/backend/fields.py

@@ -29,7 +29,7 @@ def _get_inner_types(annotation: Any) -> Generator[type, None, None]:
         yield annotation
 
 
-def _has_any_type(field: FieldInfo, *types: type) -> bool:
+def _contains_only_types(field: FieldInfo, *types: type) -> bool:
     """Return whether a field is annotated as one of the given types.
 
     Lists and unions with `NoneType` are allowed and only the non-`NoneType` annotation
@@ -71,24 +71,25 @@ def _group_fields_by_class_name(
 # fields typed as merged identifiers containing references to merged items
 REFERENCE_FIELDS_BY_CLASS_NAME = _group_fields_by_class_name(
     EXTRACTED_MODEL_CLASSES_BY_NAME,
-    lambda field_info: _has_any_type(field_info, *MERGED_IDENTIFIER_CLASSES),
+    lambda field_info: _contains_only_types(field_info, *MERGED_IDENTIFIER_CLASSES),
 )
 
 # nested fields that contain `Text` objects
 TEXT_FIELDS_BY_CLASS_NAME = _group_fields_by_class_name(
     EXTRACTED_MODEL_CLASSES_BY_NAME,
-    lambda field_info: _has_any_type(field_info, Text),
+    lambda field_info: _contains_only_types(field_info, Text),
 )
 
 # nested fields that contain `Link` objects
 LINK_FIELDS_BY_CLASS_NAME = _group_fields_by_class_name(
     EXTRACTED_MODEL_CLASSES_BY_NAME,
-    lambda field_info: _has_any_type(field_info, Link),
+    lambda field_info: _contains_only_types(field_info, Link),
 )
 
 # fields annotated as `str` type
 STRING_FIELDS_BY_CLASS_NAME = _group_fields_by_class_name(
-    EXTRACTED_MODEL_CLASSES_BY_NAME, lambda field_info: _has_any_type(field_info, str)
+    EXTRACTED_MODEL_CLASSES_BY_NAME,
+    lambda field_info: _contains_only_types(field_info, str),
 )
 
 # fields that should be indexed as searchable fields

mex/backend/graph/connector.py

@@ -18,6 +18,7 @@
 from mex.backend.graph.transform import (
     expand_references_in_search_result,
 )
+from mex.backend.settings import BackendSettings
 from mex.backend.transform import to_primitive
 from mex.common.connector import BaseConnector
 from mex.common.exceptions import MExError
@@ -55,9 +56,6 @@ def __init__(self) -> None:
 
     def _init_driver(self) -> Driver:
         """Initialize and return a database driver."""
-        # break import cycle, sigh
-        from mex.backend.settings import BackendSettings
-
         settings = BackendSettings.get()
         return GraphDatabase.driver(
             settings.graph_url,
@@ -219,10 +217,10 @@ def _merge_node(self, model: AnyExtractedModel) -> Result:
         All nested properties (like Text or Link) are created as their own nodes
         and linked via edges. For multi-valued fields, the position of each nested
         object is stored as a property on the outbound edge.
-        Any nested objects that are found in the graph, gut are not present on the
+        Any nested objects that are found in the graph, but are not present on the
         model any more are purged.
-        In addition, a merged item is created (if it does not exist yet) and linked
-        to the extracted item via an edge of the label `stableTargetId`.
+        In addition, a merged item is created (if it does not exist yet) and the
+        extracted is linked it via an edge of the label `stableTargetId`.
 
         Args:
             model: Model to merge into the graph as a node

mex/backend/graph/cypher/README.md

@@ -13,8 +13,8 @@ For example, a new model class or changing model fields are automatically handle
 and don't require rewriting any cypher query.
 
 Some of these use-cases could be covered by neo4j's [APOC](https://neo4j.com/labs/apoc/)
-extensions. However, APOC is not included in the official neo4j base image.
-So to keep deployment simple for now, the use of APOC was avoided.
+add-on (e.g. `expand_references_in_search_result`). However, APOC is not included in the
+official neo4j docker image. So, to keep deployment simple, the use of APOC was avoided.
 
 Contrary to the jinja default tags that are centered around curly braces, we use
 less/greater signs that do not collide with cypher syntax that often.

mex/backend/graph/cypher/fetch_identities.cql

@@ -15,7 +15,7 @@ Returns:
     List of identity objects.
 -#>
 MATCH (n:<<extracted_labels|join("|")>>)-[:stableTargetId]->(merged:<<merged_labels|join("|")>>)
-MATCH (n:<<extracted_labels|join("|")>>)-[:hadPrimarySource]->(primary_source:MergedPrimarySource)
+MATCH (n)-[:hadPrimarySource]->(primary_source:MergedPrimarySource)
 <%- if filter_by_had_primary_source or filter_by_identifier_in_primary_source or filter_by_stable_target_id %>
 WHERE
     <%- set and_ = joiner("AND ") -%>

mex/backend/graph/cypher/merge_edges.cql

@@ -34,9 +34,9 @@ CALL {
 WITH source, collect(edge) as edges
 CALL {
     WITH source, edges
-    MATCH (source)-[gc]->(:<<merged_labels|join("|")>>)
-    WHERE NOT gc IN edges
-    DELETE gc
-    RETURN count(gc) as pruned
+    MATCH (source)-[outdated_edge]->(:<<merged_labels|join("|")>>)
+    WHERE NOT outdated_edge IN edges
+    DELETE outdated_edge
+    RETURN count(outdated_edge) as pruned
 }
 RETURN count(edges) as merged, pruned, edges;

mex/backend/graph/cypher/merge_node.cql

@@ -39,9 +39,9 @@ WITH extracted,
     [<<range(nested_edge_labels|count)|map("ensure_prefix", "value_")|join(", ")>>] as values
 CALL {
     WITH extracted, values
-    MATCH (extracted)-[]->(gc:<<nested_labels|join("|")>>)
-    WHERE NOT gc IN values
-    DETACH DELETE gc
-    RETURN count(gc) as pruned
+    MATCH (extracted)-[]->(outdated_node:<<nested_labels|join("|")>>)
+    WHERE NOT outdated_node IN values
+    DETACH DELETE outdated_node
+    RETURN count(outdated_node) as pruned
 }
 RETURN extracted, edges, values, pruned;

mex/backend/transform.py

@@ -3,12 +3,12 @@
 
 from fastapi.encoders import jsonable_encoder
 
-from mex.common.types import Identifier, Timestamp
+from mex.common.types import Identifier, TemporalEntity
 
 JSON_ENCODERS = {
     Enum: lambda obj: obj.value,
     Identifier: lambda obj: str(obj),
-    Timestamp: lambda obj: str(obj),
+    TemporalEntity: lambda obj: str(obj),
 }