feature/mx-1764 materialize mapping models

robert-koch-institut · Jan 14, 2025 · af92478 · af92478
1 parent d900020
commit af92478
Show file tree

Hide file tree

Showing 19 changed files with 1,298 additions and 1,062 deletions.
diff --git a/mex/common/models/__init__.py b/mex/common/models/__init__.py
@@ -30,8 +30,8 @@
   contributing to specific fields of a merged item
 - `TRuleSet` classes are used for CRUD operations on a set of three rules
 
-- `ExtractedTEntityFilter` defines how an entity filter specification should look like
-- `ExtractedTMapping` defines how a raw data to extracted item mapping should look like
+- `TFilter` defines how an entity filter specification should look like
+- `TMapping` defines how a raw-data to extracted item mapping should look like
 
 Since these models for different use cases have a lot of overlapping attributes,
 we use a number of intermediate private classes to compose the public classes:
@@ -63,8 +63,8 @@
 - TRuleSetRequest: bundle of all three rules for one type used to create new rules
 - TRuleSetResponse: bundle of all three rules for one type including a `stableTargetId`
 
-- ExtractedTEntityFilter: all BaseT fields re-typed as a list of entity filters
-- ExtractedTMapping: all BaseT fields re-typed as lists of mapping fields
+- TFilter: all BaseT fields re-typed as a list of filter field definitions
+- TMapping: all BaseT fields re-typed as lists of mapping fields with `setValues` type
 
 In addition to the classes themselves, `mex.common.models` also exposes various
 lists of models, lookups by class name and typing for unions of models.
@@ -73,6 +73,8 @@
 from typing import Final, get_args
 
 from mex.common.models.access_platform import (
+    AccessPlatformFilter,
+    AccessPlatformMapping,
     AccessPlatformRuleSetRequest,
     AccessPlatformRuleSetResponse,
     AdditiveAccessPlatform,
@@ -84,6 +86,8 @@
     SubtractiveAccessPlatform,
 )
 from mex.common.models.activity import (
+    ActivityFilter,
+    ActivityMapping,
     ActivityRuleSetRequest,
     ActivityRuleSetResponse,
     AdditiveActivity,
@@ -95,14 +99,16 @@
     SubtractiveActivity,
 )
 from mex.common.models.base.extracted_data import ExtractedData
-from mex.common.models.base.filter import generate_entity_filter_schema
-from mex.common.models.base.mapping import generate_mapping_schema
+from mex.common.models.base.filter import BaseFilter, FilterField, FilterRule
+from mex.common.models.base.mapping import BaseMapping, MappingField, MappingRule
 from mex.common.models.base.merged_item import MergedItem
 from mex.common.models.base.model import BaseModel
 from mex.common.models.base.rules import AdditiveRule, PreventiveRule, SubtractiveRule
 from mex.common.models.bibliographic_resource import (
     AdditiveBibliographicResource,
     BaseBibliographicResource,
+    BibliographicResourceFilter,
+    BibliographicResourceMapping,
     BibliographicResourceRuleSetRequest,
     BibliographicResourceRuleSetResponse,
     ExtractedBibliographicResource,
@@ -114,6 +120,8 @@
 from mex.common.models.consent import (
     AdditiveConsent,
     BaseConsent,
+    ConsentFilter,
+    ConsentMapping,
     ConsentRuleSetRequest,
     ConsentRuleSetResponse,
     ExtractedConsent,
@@ -125,6 +133,8 @@
 from mex.common.models.contact_point import (
     AdditiveContactPoint,
     BaseContactPoint,
+    ContactPointFilter,
+    ContactPointMapping,
     ContactPointRuleSetRequest,
     ContactPointRuleSetResponse,
     ExtractedContactPoint,
@@ -136,6 +146,8 @@
 from mex.common.models.distribution import (
     AdditiveDistribution,
     BaseDistribution,
+    DistributionFilter,
+    DistributionMapping,
     DistributionRuleSetRequest,
     DistributionRuleSetResponse,
     ExtractedDistribution,
@@ -149,6 +161,8 @@
     BaseOrganization,
     ExtractedOrganization,
     MergedOrganization,
+    OrganizationFilter,
+    OrganizationMapping,
     OrganizationRuleSetRequest,
     OrganizationRuleSetResponse,
     PreventiveOrganization,
@@ -160,6 +174,8 @@
     BaseOrganizationalUnit,
     ExtractedOrganizationalUnit,
     MergedOrganizationalUnit,
+    OrganizationalUnitFilter,
+    OrganizationalUnitMapping,
     OrganizationalUnitRuleSetRequest,
     OrganizationalUnitRuleSetResponse,
     PreventiveOrganizationalUnit,
@@ -171,6 +187,8 @@
     BasePerson,
     ExtractedPerson,
     MergedPerson,
+    PersonFilter,
+    PersonMapping,
     PersonRuleSetRequest,
     PersonRuleSetResponse,
     PreventivePerson,
@@ -184,6 +202,8 @@
     MergedPrimarySource,
     PreventivePrimarySource,
     PreviewPrimarySource,
+    PrimarySourceFilter,
+    PrimarySourceMapping,
     PrimarySourceRuleSetRequest,
     PrimarySourceRuleSetResponse,
     SubtractivePrimarySource,
@@ -195,6 +215,8 @@
     MergedResource,
     PreventiveResource,
     PreviewResource,
+    ResourceFilter,
+    ResourceMapping,
     ResourceRuleSetRequest,
     ResourceRuleSetResponse,
     SubtractiveResource,
@@ -207,6 +229,8 @@
     PreventiveVariable,
     PreviewVariable,
     SubtractiveVariable,
+    VariableFilter,
+    VariableMapping,
     VariableRuleSetRequest,
     VariableRuleSetResponse,
 )
@@ -218,6 +242,8 @@
     PreventiveVariableGroup,
     PreviewVariableGroup,
     SubtractiveVariableGroup,
+    VariableGroupFilter,
+    VariableGroupMapping,
     VariableGroupRuleSetRequest,
     VariableGroupRuleSetResponse,
 )
@@ -250,8 +276,12 @@
     "RULE_SET_RESPONSE_CLASSES_BY_NAME",
     "SUBTRACTIVE_MODEL_CLASSES",
     "SUBTRACTIVE_MODEL_CLASSES_BY_NAME",
+    "AccessPlatformFilter",
+    "AccessPlatformMapping",
     "AccessPlatformRuleSetRequest",
     "AccessPlatformRuleSetResponse",
+    "ActivityFilter",
+    "ActivityMapping",
     "ActivityRuleSetRequest",
     "ActivityRuleSetResponse",
     "AdditiveAccessPlatform",
@@ -283,6 +313,8 @@
     "BaseConsent",
     "BaseContactPoint",
     "BaseDistribution",
+    "BaseFilter",
+    "BaseMapping",
     "BaseModel",
     "BaseOrganization",
     "BaseOrganizationalUnit",
@@ -291,10 +323,18 @@
     "BaseResource",
     "BaseVariable",
     "BaseVariableGroup",
+    "BibliographicResourceFilter",
+    "BibliographicResourceMapping",
+    "ConsentFilter",
+    "ConsentMapping",
     "ConsentRuleSetRequest",
     "ConsentRuleSetResponse",
+    "ContactPointFilter",
+    "ContactPointMapping",
     "ContactPointRuleSetRequest",
     "ContactPointRuleSetResponse",
+    "DistributionFilter",
+    "DistributionMapping",
     "DistributionRuleSetRequest",
     "DistributionRuleSetResponse",
     "ExtractedAccessPlatform",
@@ -311,6 +351,10 @@
     "ExtractedResource",
     "ExtractedVariable",
     "ExtractedVariableGroup",
+    "FilterField",
+    "FilterRule",
+    "MappingField",
+    "MappingRule",
     "MergedAccessPlatform",
     "MergedActivity",
     "MergedBibliographicResource",
@@ -325,10 +369,16 @@
     "MergedResource",
     "MergedVariable",
     "MergedVariableGroup",
+    "OrganizationFilter",
+    "OrganizationMapping",
     "OrganizationRuleSetRequest",
     "OrganizationRuleSetResponse",
+    "OrganizationalUnitFilter",
+    "OrganizationalUnitMapping",
     "OrganizationalUnitRuleSetRequest",
     "OrganizationalUnitRuleSetResponse",
+    "PersonFilter",
+    "PersonMapping",
     "PersonRuleSetRequest",
     "PersonRuleSetResponse",
     "PreventiveAccessPlatform",
@@ -358,8 +408,12 @@
     "PreviewResource",
     "PreviewVariable",
     "PreviewVariableGroup",
+    "PrimarySourceFilter",
+    "PrimarySourceMapping",
     "PrimarySourceRuleSetRequest",
     "PrimarySourceRuleSetResponse",
+    "ResourceFilter",
+    "ResourceMapping",
     "ResourceRuleSetRequest",
     "ResourceRuleSetResponse",
     "SubtractiveAccessPlatform",
@@ -376,8 +430,12 @@
     "SubtractiveRule",
     "SubtractiveVariable",
     "SubtractiveVariableGroup",
+    "VariableFilter",
+    "VariableGroupFilter",
+    "VariableGroupMapping",
     "VariableGroupRuleSetRequest",
     "VariableGroupRuleSetResponse",
+    "VariableMapping",
     "VariableRuleSetRequest",
     "VariableRuleSetResponse",
 )
@@ -586,10 +644,52 @@
     cls.__name__: cls for cls in RULE_SET_RESPONSE_CLASSES
 }
 
-FILTER_MODEL_BY_EXTRACTED_CLASS_NAME = {
-    cls.__name__: generate_entity_filter_schema(cls) for cls in EXTRACTED_MODEL_CLASSES
+AnyMappingModel = (
+    AccessPlatformMapping
+    | ActivityMapping
+    | BibliographicResourceMapping
+    | ConsentMapping
+    | ContactPointMapping
+    | DistributionMapping
+    | OrganizationMapping
+    | OrganizationalUnitMapping
+    | PersonMapping
+    | PrimarySourceMapping
+    | ResourceMapping
+    | VariableMapping
+    | VariableGroupMapping
+)
+MAPPING_MODEL_CLASSES: Final[list[type[AnyMappingModel]]] = list(
+    get_args(AnyMappingModel)
+)
+MAPPING_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyMappingModel]]] = {
+    cls.__name__: cls for cls in MAPPING_MODEL_CLASSES
 }
-
+# MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME is deprecated, use MAPPING_MODEL_CLASSES_BY_NAME
 MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME = {
-    cls.__name__: generate_mapping_schema(cls) for cls in EXTRACTED_MODEL_CLASSES
+    f"Extracted{cls.stemType}": cls for cls in MAPPING_MODEL_CLASSES
+}
+
+AnyFilterModel = (
+    AccessPlatformFilter
+    | ActivityFilter
+    | BibliographicResourceFilter
+    | ConsentFilter
+    | ContactPointFilter
+    | DistributionFilter
+    | OrganizationFilter
+    | OrganizationalUnitFilter
+    | PersonFilter
+    | PrimarySourceFilter
+    | ResourceFilter
+    | VariableFilter
+    | VariableGroupFilter
+)
+FILTER_MODEL_CLASSES: Final[list[type[AnyFilterModel]]] = list(get_args(AnyFilterModel))
+FILTER_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyFilterModel]]] = {
+    cls.__name__: cls for cls in FILTER_MODEL_CLASSES
+}
+# FILTER_MODEL_BY_EXTRACTED_CLASS_NAME is deprecated, use FILTER_MODEL_CLASSES_BY_NAME
+FILTER_MODEL_BY_EXTRACTED_CLASS_NAME = {
+    f"Extracted{cls.stemType}": cls for cls in FILTER_MODEL_CLASSES
 }
diff --git a/mex/common/models/access_platform.py b/mex/common/models/access_platform.py
@@ -5,6 +5,8 @@
 from pydantic import AfterValidator, Field, computed_field
 
 from mex.common.models.base.extracted_data import ExtractedData
+from mex.common.models.base.filter import BaseFilter, FilterField
+from mex.common.models.base.mapping import BaseMapping, MappingField
 from mex.common.models.base.merged_item import MergedItem
 from mex.common.models.base.model import BaseModel
 from mex.common.models.base.preview_item import PreviewItem
@@ -28,6 +30,13 @@
     Text,
 )
 
+AnyContactIdentifier = Annotated[
+    MergedOrganizationalUnitIdentifier
+    | MergedPersonIdentifier
+    | MergedContactPointIdentifier,
+    AfterValidator(Identifier),
+]
+
 
 class _Stem(BaseModel):
     stemType: ClassVar[Annotated[Literal["AccessPlatform"], Field(frozen=True)]] = (
@@ -37,14 +46,7 @@ class _Stem(BaseModel):
 
 class _OptionalLists(_Stem):
     alternativeTitle: list[Text] = []
-    contact: list[
-        Annotated[
-            MergedOrganizationalUnitIdentifier
-            | MergedPersonIdentifier
-            | MergedContactPointIdentifier,
-            AfterValidator(Identifier),
-        ]
-    ] = []
+    contact: list[AnyContactIdentifier] = []
     description: list[Text] = []
     landingPage: list[Link] = []
     title: list[Text] = []
@@ -173,3 +175,47 @@ class AccessPlatformRuleSetResponse(_BaseRuleSet):
         Literal["AccessPlatformRuleSetResponse"], Field(alias="$type", frozen=True)
     ] = "AccessPlatformRuleSetResponse"
     stableTargetId: MergedAccessPlatformIdentifier
+
+
+class AccessPlatformMapping(_Stem, BaseMapping):
+    """Mapping for describing an access platform transformation."""
+
+    entityType: Annotated[
+        Literal["AccessPlatformMapping"], Field(alias="$type", frozen=True)
+    ] = "AccessPlatformMapping"
+    hadPrimarySource: Annotated[
+        list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1)
+    ]
+    identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)]
+    technicalAccessibility: Annotated[
+        list[MappingField[TechnicalAccessibility]], Field(min_length=1)
+    ]
+    endpointDescription: list[MappingField[Link | None]] = []
+    endpointType: list[MappingField[APIType | None]] = []
+    endpointURL: list[MappingField[Link | None]] = []
+    alternativeTitle: list[MappingField[list[Text]]] = []
+    contact: list[MappingField[list[AnyContactIdentifier]]] = []
+    description: list[MappingField[list[Text]]] = []
+    landingPage: list[MappingField[list[Link]]] = []
+    title: list[MappingField[list[Text]]] = []
+    unitInCharge: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = []
+
+
+class AccessPlatformFilter(_Stem, BaseFilter):
+    """Class for defining filter rules for access platform items."""
+
+    entityType: Annotated[
+        Literal["AccessPlatformFilter"], Field(alias="$type", frozen=True)
+    ] = "AccessPlatformFilter"
+    hadPrimarySource: list[FilterField] = []
+    identifierInPrimarySource: list[FilterField] = []
+    alternativeTitle: list[FilterField] = []
+    contact: list[FilterField] = []
+    description: list[FilterField] = []
+    endpointDescription: list[FilterField] = []
+    endpointType: list[FilterField] = []
+    endpointURL: list[FilterField] = []
+    landingPage: list[FilterField] = []
+    technicalAccessibility: list[FilterField] = []
+    title: list[FilterField] = []
+    unitInCharge: list[FilterField] = []