feature/mx-1764 materialize mapping models (#364)

### Added - new (partially generic) classes for defining Mapping and Filter fields and rules ### Changes - BREAKING: replaced dynamic Mapping and Filter classes with static ones ### Deprecated - use FILTER_MODEL_CLASSES_BY_NAME instead of FILTER_MODEL_BY_EXTRACTED_CLASS_NAME - use MAPPING_MODEL_CLASSES_BY_NAME instead of MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME --------- Signed-off-by: Nicolas Drebenstedt <[email protected]> Co-authored-by: erichesse <[email protected]>
robert-koch-institut · Jan 29, 2025 · 40dcfe6 · 40dcfe6
1 parent 9a91777
commit 40dcfe6
Show file tree

Hide file tree

Showing 20 changed files with 1,317 additions and 1,015 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,10 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- new (partially generic) classes for defining Mapping and Filter fields and rules
+
 ### Changes
 
+- BREAKING: replaced dynamic Mapping and Filter classes with static ones
+
 ### Deprecated
 
+- use FILTER_MODEL_CLASSES_BY_NAME instead of FILTER_MODEL_BY_EXTRACTED_CLASS_NAME
+- use MAPPING_MODEL_CLASSES_BY_NAME instead of MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME
+
 ### Removed
 
 ### Fixed

diff --git a/mex/common/models/__init__.py b/mex/common/models/__init__.py
@@ -30,8 +30,8 @@
   contributing to specific fields of a merged item
 - `TRuleSet` classes are used for CRUD operations on a set of three rules
 
-- `ExtractedTEntityFilter` defines how an entity filter specification should look like
-- `ExtractedTMapping` defines how a raw data to extracted item mapping should look like
+- `TFilter` defines how an entity filter specification should look like
+- `TMapping` defines how a raw-data to extracted item mapping should look like
 
 Since these models for different use cases have a lot of overlapping attributes,
 we use a number of intermediate private classes to compose the public classes:
@@ -63,8 +63,8 @@
 - TRuleSetRequest: bundle of all three rules for one type used to create new rules
 - TRuleSetResponse: bundle of all three rules for one type including a `stableTargetId`
 
-- ExtractedTEntityFilter: all BaseT fields re-typed as a list of EntityFilter
-- ExtractedTMapping: all BaseT fields re-typed as lists of subclasses of GenericField
+- TFilter: all BaseT fields re-typed as a list of filter field definitions
+- TMapping: all BaseT fields re-typed as lists of mapping fields with `setValues` type
 
 In addition to the classes themselves, `mex.common.models` also exposes various
 lists of models, lookups by class name and typing for unions of models.
@@ -73,6 +73,8 @@
 from typing import Final, get_args
 
 from mex.common.models.access_platform import (
+    AccessPlatformFilter,
+    AccessPlatformMapping,
     AccessPlatformRuleSetRequest,
     AccessPlatformRuleSetResponse,
     AdditiveAccessPlatform,
@@ -84,6 +86,8 @@
     SubtractiveAccessPlatform,
 )
 from mex.common.models.activity import (
+    ActivityFilter,
+    ActivityMapping,
     ActivityRuleSetRequest,
     ActivityRuleSetResponse,
     AdditiveActivity,
@@ -95,14 +99,16 @@
     SubtractiveActivity,
 )
 from mex.common.models.base.extracted_data import ExtractedData
-from mex.common.models.base.filter import generate_entity_filter_schema
-from mex.common.models.base.mapping import generate_mapping_schema
+from mex.common.models.base.filter import BaseFilter, FilterField, FilterRule
+from mex.common.models.base.mapping import BaseMapping, MappingField, MappingRule
 from mex.common.models.base.merged_item import MergedItem
 from mex.common.models.base.model import BaseModel
 from mex.common.models.base.rules import AdditiveRule, PreventiveRule, SubtractiveRule
 from mex.common.models.bibliographic_resource import (
     AdditiveBibliographicResource,
     BaseBibliographicResource,
+    BibliographicResourceFilter,
+    BibliographicResourceMapping,
     BibliographicResourceRuleSetRequest,
     BibliographicResourceRuleSetResponse,
     ExtractedBibliographicResource,
@@ -114,6 +120,8 @@
 from mex.common.models.consent import (
     AdditiveConsent,
     BaseConsent,
+    ConsentFilter,
+    ConsentMapping,
     ConsentRuleSetRequest,
     ConsentRuleSetResponse,
     ExtractedConsent,
@@ -125,6 +133,8 @@
 from mex.common.models.contact_point import (
     AdditiveContactPoint,
     BaseContactPoint,
+    ContactPointFilter,
+    ContactPointMapping,
     ContactPointRuleSetRequest,
     ContactPointRuleSetResponse,
     ExtractedContactPoint,
@@ -136,6 +146,8 @@
 from mex.common.models.distribution import (
     AdditiveDistribution,
     BaseDistribution,
+    DistributionFilter,
+    DistributionMapping,
     DistributionRuleSetRequest,
     DistributionRuleSetResponse,
     ExtractedDistribution,
@@ -149,6 +161,8 @@
     BaseOrganization,
     ExtractedOrganization,
     MergedOrganization,
+    OrganizationFilter,
+    OrganizationMapping,
     OrganizationRuleSetRequest,
     OrganizationRuleSetResponse,
     PreventiveOrganization,
@@ -160,6 +174,8 @@
     BaseOrganizationalUnit,
     ExtractedOrganizationalUnit,
     MergedOrganizationalUnit,
+    OrganizationalUnitFilter,
+    OrganizationalUnitMapping,
     OrganizationalUnitRuleSetRequest,
     OrganizationalUnitRuleSetResponse,
     PreventiveOrganizationalUnit,
@@ -171,6 +187,8 @@
     BasePerson,
     ExtractedPerson,
     MergedPerson,
+    PersonFilter,
+    PersonMapping,
     PersonRuleSetRequest,
     PersonRuleSetResponse,
     PreventivePerson,
@@ -184,6 +202,8 @@
     MergedPrimarySource,
     PreventivePrimarySource,
     PreviewPrimarySource,
+    PrimarySourceFilter,
+    PrimarySourceMapping,
     PrimarySourceRuleSetRequest,
     PrimarySourceRuleSetResponse,
     SubtractivePrimarySource,
@@ -195,6 +215,8 @@
     MergedResource,
     PreventiveResource,
     PreviewResource,
+    ResourceFilter,
+    ResourceMapping,
     ResourceRuleSetRequest,
     ResourceRuleSetResponse,
     SubtractiveResource,
@@ -207,6 +229,8 @@
     PreventiveVariable,
     PreviewVariable,
     SubtractiveVariable,
+    VariableFilter,
+    VariableMapping,
     VariableRuleSetRequest,
     VariableRuleSetResponse,
 )
@@ -218,6 +242,8 @@
     PreventiveVariableGroup,
     PreviewVariableGroup,
     SubtractiveVariableGroup,
+    VariableGroupFilter,
+    VariableGroupMapping,
     VariableGroupRuleSetRequest,
     VariableGroupRuleSetResponse,
 )
@@ -250,8 +276,12 @@
     "RULE_SET_RESPONSE_CLASSES_BY_NAME",
     "SUBTRACTIVE_MODEL_CLASSES",
     "SUBTRACTIVE_MODEL_CLASSES_BY_NAME",
+    "AccessPlatformFilter",
+    "AccessPlatformMapping",
     "AccessPlatformRuleSetRequest",
     "AccessPlatformRuleSetResponse",
+    "ActivityFilter",
+    "ActivityMapping",
     "ActivityRuleSetRequest",
     "ActivityRuleSetResponse",
     "AdditiveAccessPlatform",
@@ -283,6 +313,8 @@
     "BaseConsent",
     "BaseContactPoint",
     "BaseDistribution",
+    "BaseFilter",
+    "BaseMapping",
     "BaseModel",
     "BaseOrganization",
     "BaseOrganizationalUnit",
@@ -291,10 +323,18 @@
     "BaseResource",
     "BaseVariable",
     "BaseVariableGroup",
+    "BibliographicResourceFilter",
+    "BibliographicResourceMapping",
+    "ConsentFilter",
+    "ConsentMapping",
     "ConsentRuleSetRequest",
     "ConsentRuleSetResponse",
+    "ContactPointFilter",
+    "ContactPointMapping",
     "ContactPointRuleSetRequest",
     "ContactPointRuleSetResponse",
+    "DistributionFilter",
+    "DistributionMapping",
     "DistributionRuleSetRequest",
     "DistributionRuleSetResponse",
     "ExtractedAccessPlatform",
@@ -311,6 +351,10 @@
     "ExtractedResource",
     "ExtractedVariable",
     "ExtractedVariableGroup",
+    "FilterField",
+    "FilterRule",
+    "MappingField",
+    "MappingRule",
     "MergedAccessPlatform",
     "MergedActivity",
     "MergedBibliographicResource",
@@ -325,10 +369,16 @@
     "MergedResource",
     "MergedVariable",
     "MergedVariableGroup",
+    "OrganizationFilter",
+    "OrganizationMapping",
     "OrganizationRuleSetRequest",
     "OrganizationRuleSetResponse",
+    "OrganizationalUnitFilter",
+    "OrganizationalUnitMapping",
     "OrganizationalUnitRuleSetRequest",
     "OrganizationalUnitRuleSetResponse",
+    "PersonFilter",
+    "PersonMapping",
     "PersonRuleSetRequest",
     "PersonRuleSetResponse",
     "PreventiveAccessPlatform",
@@ -358,8 +408,12 @@
     "PreviewResource",
     "PreviewVariable",
     "PreviewVariableGroup",
+    "PrimarySourceFilter",
+    "PrimarySourceMapping",
     "PrimarySourceRuleSetRequest",
     "PrimarySourceRuleSetResponse",
+    "ResourceFilter",
+    "ResourceMapping",
     "ResourceRuleSetRequest",
     "ResourceRuleSetResponse",
     "SubtractiveAccessPlatform",
@@ -376,8 +430,12 @@
     "SubtractiveRule",
     "SubtractiveVariable",
     "SubtractiveVariableGroup",
+    "VariableFilter",
+    "VariableGroupFilter",
+    "VariableGroupMapping",
     "VariableGroupRuleSetRequest",
     "VariableGroupRuleSetResponse",
+    "VariableMapping",
     "VariableRuleSetRequest",
     "VariableRuleSetResponse",
 )
@@ -586,10 +644,52 @@
     cls.__name__: cls for cls in RULE_SET_RESPONSE_CLASSES
 }
 
-FILTER_MODEL_BY_EXTRACTED_CLASS_NAME = {
-    cls.__name__: generate_entity_filter_schema(cls) for cls in EXTRACTED_MODEL_CLASSES
+AnyMappingModel = (
+    AccessPlatformMapping
+    | ActivityMapping
+    | BibliographicResourceMapping
+    | ConsentMapping
+    | ContactPointMapping
+    | DistributionMapping
+    | OrganizationMapping
+    | OrganizationalUnitMapping
+    | PersonMapping
+    | PrimarySourceMapping
+    | ResourceMapping
+    | VariableMapping
+    | VariableGroupMapping
+)
+MAPPING_MODEL_CLASSES: Final[list[type[AnyMappingModel]]] = list(
+    get_args(AnyMappingModel)
+)
+MAPPING_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyMappingModel]]] = {
+    cls.__name__: cls for cls in MAPPING_MODEL_CLASSES
 }
-
+# MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME is deprecated, use MAPPING_MODEL_CLASSES_BY_NAME
 MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME = {
-    cls.__name__: generate_mapping_schema(cls) for cls in EXTRACTED_MODEL_CLASSES
+    f"Extracted{cls.stemType}": cls for cls in MAPPING_MODEL_CLASSES
+}
+
+AnyFilterModel = (
+    AccessPlatformFilter
+    | ActivityFilter
+    | BibliographicResourceFilter
+    | ConsentFilter
+    | ContactPointFilter
+    | DistributionFilter
+    | OrganizationFilter
+    | OrganizationalUnitFilter
+    | PersonFilter
+    | PrimarySourceFilter
+    | ResourceFilter
+    | VariableFilter
+    | VariableGroupFilter
+)
+FILTER_MODEL_CLASSES: Final[list[type[AnyFilterModel]]] = list(get_args(AnyFilterModel))
+FILTER_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyFilterModel]]] = {
+    cls.__name__: cls for cls in FILTER_MODEL_CLASSES
+}
+# FILTER_MODEL_BY_EXTRACTED_CLASS_NAME is deprecated, use FILTER_MODEL_CLASSES_BY_NAME
+FILTER_MODEL_BY_EXTRACTED_CLASS_NAME = {
+    f"Extracted{cls.stemType}": cls for cls in FILTER_MODEL_CLASSES
 }