feature/mx-1381 prepare database model rework

.cruft.json

@@ -1,6 +1,6 @@
 {
   "checkout": null,
-  "commit": "6067fc53d1335a9bda900c5eff8dbf1c42bfe4ca",
+  "commit": "8a99c5bda13e8a909b26d195886225f5d278e693",
   "context": {
     "cookiecutter": {
       "project_name": "common",

CHANGELOG.md

@@ -9,12 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- add `entityType` type hint to `MExModel` (now `BaseEntity`)
+- add types for `AnyBaseModel`, `AnyExtractedModel` and `AnyMergedModel`
+- create more specific subclasses of `Identifier` (for extracted and merged)
+- expose unions, lists and lookups for `Identifier` subclasses in `mex.common.types`
+
 ### Changes
 
+- swap `contextvars.ContextVar` for `mex.common.context.ContextStore`
+- move `stableTargetId` property from base models to extracted models
+- update typing of identifiers to specific subclasses
+- use `Annotated[..., Field(...)]` notation for pydantic field configs
+- split up `mex.common.models.base` and move out `MExModel` and `JsonSchemaGenerator`
+- rename `MExModel` to `BaseEntity` with only type hints an model config
+- declare `hadPrimarySource`, `identifier` and `identifierInPrimarySource` as frozen
+
 ### Deprecated
 
 ### Removed
 
+- absorb unused `BaseExtractedData` into `ExtractedData`
+- remove `stableTargetId` property from merged models
+- drop support for sinks to accept merged items (now only for extracted data)
+
 ### Fixed
 
 ### Security

mex/common/backend_api/connector.py

@@ -3,7 +3,7 @@
 
 from mex.common.backend_api.models import BulkInsertResponse
 from mex.common.connector import HTTPConnector
-from mex.common.models import MExModel
+from mex.common.models import ExtractedData
 from mex.common.settings import BaseSettings
 from mex.common.types import Identifier
 
@@ -27,7 +27,7 @@ def _set_url(self) -> None:
         settings = BaseSettings.get()
         self.url = urljoin(str(settings.backend_api_url), self.API_VERSION)
 
-    def post_models(self, models: list[MExModel]) -> list[Identifier]:
+    def post_models(self, models: list[ExtractedData]) -> list[Identifier]:
         """Post models to Backend API in a bulk insertion request.
 
         Args:
@@ -37,7 +37,7 @@ def post_models(self, models: list[MExModel]) -> list[Identifier]:
             HTTPError: If insert was not accepted, crashes or times out
 
         Returns:
-            Identifiers of posted models
+            Identifiers of posted extracted models
         """
         response = self.request(
             method="POST",

mex/common/cli.py

@@ -116,7 +116,7 @@ def _callback(
     # ensure connectors are closed on exit.
     context.call_on_close(reset_connector_context)
 
-    # load settings from parameters and store in ContextVar.
+    # load settings from parameters and store it globally.
     settings = settings_cls.model_validate(
         {
             key: value
@@ -126,7 +126,7 @@ def _callback(
     )
     SettingsContext.set(settings)
 
-    # otherwise print loaded settings in pretty way and continue
+    # otherwise print loaded settings in pretty way and continue.
     logger.info(click.style(dedent(f"    {func.__doc__}"), fg="green"))
     logger.info(click.style(f"{settings.text()}\n", fg="bright_cyan"))
 
@@ -142,7 +142,7 @@ def _callback(
             # if we are in debug mode, jump into interactive debugging.
             pdb.post_mortem(sys.exc_info()[2])
             raise error
-        # if not in debug mode, exit with code 1
+        # if not in debug mode, exit with code 1.
         echo("exit", fg="red")
         context.exit(1)
 

mex/common/connector/base.py

@@ -1,14 +1,12 @@
 from abc import ABCMeta, abstractmethod
 from contextlib import ExitStack
-from contextvars import ContextVar
 from types import TracebackType
 from typing import Optional, TypeVar, cast, final
 
+from mex.common.context import ContextStore
+
 ConnectorType = TypeVar("ConnectorType", bound="BaseConnector")
-ConnectorContextType = dict[type["BaseConnector"], "BaseConnector"]
-ConnectorContext = ContextVar(
-    "ConnectorContext", default=cast(ConnectorContextType, {})
-)
+ConnectorContext = ContextStore[dict[type["BaseConnector"], "BaseConnector"]]({})
 
 
 def reset_connector_context() -> None:

mex/common/context.py

@@ -0,0 +1,19 @@
+from typing import Generic, TypeVar
+
+ContextResource = TypeVar("ContextResource")
+
+
+class ContextStore(Generic[ContextResource]):
+    """Thin wrapper for storing thread-local globals."""
+
+    def __init__(self, default: ContextResource) -> None:
+        """Create a new context store with a default value."""
+        self._resource = default
+
+    def get(self) -> ContextResource:
+        """Retrieve the current value stored in this context."""
+        return self._resource
+
+    def set(self, resource: ContextResource) -> None:
+        """Update the current value stored in this context."""
+        self._resource = resource

mex/common/identity/base.py

@@ -2,7 +2,7 @@
 
 from mex.common.connector import BaseConnector
 from mex.common.identity.models import Identity
-from mex.common.types import Identifier, PrimarySourceID
+from mex.common.types import AnyMergedIdentifier, MergedPrimarySourceIdentifier
 
 
 class BaseProvider(BaseConnector):
@@ -11,7 +11,7 @@ class BaseProvider(BaseConnector):
     @abstractmethod
     def assign(
         self,
-        had_primary_source: PrimarySourceID,
+        had_primary_source: MergedPrimarySourceIdentifier,
         identifier_in_primary_source: str,
     ) -> Identity:  # pragma: no cover
         """Find an Identity in a database or assign a new one."""
@@ -21,9 +21,9 @@ def assign(
     def fetch(
         self,
         *,
-        had_primary_source: Identifier | None = None,
+        had_primary_source: MergedPrimarySourceIdentifier | None = None,
         identifier_in_primary_source: str | None = None,
-        stable_target_id: Identifier | None = None,
+        stable_target_id: AnyMergedIdentifier | None = None,
     ) -> list[Identity]:  # pragma: no cover
         """Find Identity instances matching the given filters."""
         ...

mex/common/identity/memory.py

@@ -4,7 +4,11 @@
     MEX_PRIMARY_SOURCE_IDENTIFIER_IN_PRIMARY_SOURCE,
     MEX_PRIMARY_SOURCE_STABLE_TARGET_ID,
 )
-from mex.common.types import Identifier, PrimarySourceID
+from mex.common.types import (
+    AnyMergedIdentifier,
+    Identifier,
+    MergedPrimarySourceIdentifier,
+)
 
 
 class MemoryIdentityProvider(BaseProvider):
@@ -22,7 +26,9 @@ def __init__(self) -> None:
         ]
 
     def assign(
-        self, had_primary_source: PrimarySourceID, identifier_in_primary_source: str
+        self,
+        had_primary_source: MergedPrimarySourceIdentifier,
+        identifier_in_primary_source: str,
     ) -> Identity:
         """Find an Identity in the in-memory database or assign a new one.
 
@@ -52,9 +58,9 @@ def assign(
     def fetch(
         self,
         *,
-        had_primary_source: Identifier | None = None,
+        had_primary_source: MergedPrimarySourceIdentifier | None = None,
         identifier_in_primary_source: str | None = None,
-        stable_target_id: Identifier | None = None,
+        stable_target_id: AnyMergedIdentifier | None = None,
     ) -> list[Identity]:
         """Find Identity instances in the in-memory database.
 

mex/common/identity/models.py

@@ -1,11 +1,15 @@
+from typing import Annotated
+
+from pydantic import Field
+
 from mex.common.models import BaseModel
-from mex.common.types import Identifier, PrimarySourceID
+from mex.common.types import Identifier, MergedPrimarySourceIdentifier
 
 
 class Identity(BaseModel):
     """Model for identifier lookup."""
 
-    identifier: Identifier
-    hadPrimarySource: PrimarySourceID
-    identifierInPrimarySource: str
-    stableTargetId: Identifier
+    identifier: Annotated[Identifier, Field(frozen=True)]
+    hadPrimarySource: Annotated[MergedPrimarySourceIdentifier, Field(frozen=True)]
+    identifierInPrimarySource: Annotated[str, Field(frozen=True)]
+    stableTargetId: Annotated[Identifier, Field(frozen=True)]

mex/common/ldap/README.md

@@ -23,7 +23,7 @@ The module `ldap.transform` contains functions for transforming LDAP data into M
 models.
 
 The `mex_person.stableTargetId` attribute can be used in any entity that requires a
-`PersonID`.
+`MergedPersonIdentifier`.
 
 # Convenience Functions
 

mex/common/ldap/extract.py

@@ -4,17 +4,17 @@
 from mex.common.identity import get_provider
 from mex.common.ldap.models.person import LDAPPerson, LDAPPersonWithQuery
 from mex.common.models import ExtractedPrimarySource
-from mex.common.types import PersonID
+from mex.common.types import MergedPersonIdentifier
 
 
 def _get_merged_ids_by_attribute(
     attribute: str,
     persons: Iterable[LDAPPerson],
     primary_source: ExtractedPrimarySource,
-) -> dict[str, list[PersonID]]:
-    """Return a mapping from a dynamic Person attribute to corresponding PersonIDs.
+) -> dict[str, list[MergedPersonIdentifier]]:
+    """Return mapping from dynamic Person attribute to corresponding merged person ids.
 
-    PersonIDs are looked up in the identity provider and will be omitted
+    MergedPersonIdentifiers are looked up in the identity provider and will be omitted
     for any person that has not yet been assigned an `Identity` there.
 
     Args:
@@ -23,7 +23,8 @@ def _get_merged_ids_by_attribute(
         primary_source: Primary source for LDAP
 
     Returns:
-        Mapping from a stringified `LDAPPerson[attribute]` to corresponding PersonIDs
+        Mapping from a stringified `LDAPPerson[attribute]` to corresponding
+        MergedPersonIdentifiers
     """
     if attribute not in LDAPPerson.model_fields:
         raise RuntimeError(f"Not a valid LDAPPerson field: {attribute}")
@@ -35,62 +36,63 @@ def _get_merged_ids_by_attribute(
             identifier_in_primary_source=str(person.objectGUID),
         ):
             merged_ids_by_attribute[str(getattr(person, attribute))].append(
-                PersonID(identities[0].stableTargetId)
+                MergedPersonIdentifier(identities[0].stableTargetId)
             )
     return merged_ids_by_attribute
 
 
 def get_merged_ids_by_employee_ids(
     persons: Iterable[LDAPPerson], primary_source: ExtractedPrimarySource
-) -> dict[str, list[PersonID]]:
-    """Return a mapping from a person's employeeID to their PersonIDs.
+) -> dict[str, list[MergedPersonIdentifier]]:
+    """Return a mapping from a person's employeeID to their merged person ids.
 
-    PersonIDs are looked up in the identity provider and will be omitted
+    MergedPersonIdentifiers are looked up in the identity provider and will be omitted
     for any person that has not yet been assigned an `Identity` there.
 
     Args:
         persons: Iterable of LDAP persons
         primary_source: Primary source for LDAP
 
     Returns:
-        Mapping from `LDAPPerson.employeeID` to corresponding PersonIDs
+        Mapping from `LDAPPerson.employeeID` to corresponding MergedPersonIdentifiers
     """
     return _get_merged_ids_by_attribute("employeeID", persons, primary_source)
 
 
 def get_merged_ids_by_email(
     persons: Iterable[LDAPPerson], primary_source: ExtractedPrimarySource
-) -> dict[str, list[PersonID]]:
-    """Return a mapping from a person's e-mail to their PersonIDs.
+) -> dict[str, list[MergedPersonIdentifier]]:
+    """Return a mapping from a person's e-mail to their merged person ids.
 
-    PersonIDs are looked up in the identity provider and will be omitted
+    MergedPersonIdentifiers are looked up in the identity provider and will be omitted
     for any person that has not yet been assigned an `Identity` there.
 
     Args:
         persons: Iterable of LDP persons
         primary_source: Primary source for LDAP
 
     Returns:
-        Mapping from `LDAPPerson.mail` to corresponding PersonIDs
+        Mapping from `LDAPPerson.mail` to corresponding MergedPersonIdentifiers
     """
     return _get_merged_ids_by_attribute("mail", persons, primary_source)
 
 
 def get_merged_ids_by_query_string(
     persons_with_query: Iterable[LDAPPersonWithQuery],
     primary_source: ExtractedPrimarySource,
-) -> dict[str, list[PersonID]]:
-    """Return a mapping from a person query string to their PersonIDs.
+) -> dict[str, list[MergedPersonIdentifier]]:
+    """Return a mapping from a person query string to their merged person ids.
 
-    PersonIDs are looked up in the identity provider and will be omitted
+    MergedPersonIdentifiers are looked up in the identity provider and will be omitted
     for any person that has not yet been assigned an `Identity` there.
 
     Args:
         persons_with_query: Iterable of LDP persons with query
         primary_source: Primary source for LDAP
 
     Returns:
-        Mapping from `LDAPPersonWithQuery.query` to corresponding PersonIDs
+        Mapping from `LDAPPersonWithQuery.query` to corresponding
+        MergedPersonIdentifiers
     """
     merged_ids_by_attribute = defaultdict(list)
     provider = get_provider()
@@ -100,6 +102,6 @@ def get_merged_ids_by_query_string(
             identifier_in_primary_source=str(person_with_query.person.objectGUID),
         ):
             merged_ids_by_attribute[str(person_with_query.query)].append(
-                PersonID(identities[0].stableTargetId)
+                MergedPersonIdentifier(identities[0].stableTargetId)
             )
     return merged_ids_by_attribute

mex/common/ldap/models/person.py

@@ -1,3 +1,5 @@
+from typing import Annotated
+
 from pydantic import Field
 
 from mex.common.ldap.models.actor import LDAPActor
@@ -12,7 +14,7 @@ class LDAPPerson(LDAPActor):
     departmentNumber: str | None = None
     displayName: str | None = None
     employeeID: str
-    givenName: list[str] = Field(min_length=1)
+    givenName: Annotated[list[str], Field(min_length=1)]
     ou: list[str] = []
     sn: str