From af92478e2cdcec6e8b805b107771f60daeccee17 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Tue, 14 Jan 2025 15:32:54 +0100 Subject: [PATCH] feature/mx-1764 materialize mapping models --- mex/common/models/__init__.py | 120 +++- mex/common/models/access_platform.py | 62 +- mex/common/models/activity.py | 108 ++- mex/common/models/base/filter.py | 64 +- mex/common/models/base/mapping.py | 133 +--- mex/common/models/bibliographic_resource.py | 102 ++- mex/common/models/consent.py | 36 + mex/common/models/contact_point.py | 26 + mex/common/models/distribution.py | 49 ++ mex/common/models/organization.py | 157 +++-- mex/common/models/organizational_unit.py | 38 ++ mex/common/models/person.py | 124 ++-- mex/common/models/primary_source.py | 81 ++- mex/common/models/resource.py | 241 +++++-- mex/common/models/variable.py | 91 ++- mex/common/models/variable_group.py | 30 + tests/models/test_filter.py | 171 +++-- tests/models/test_mapping.py | 713 ++++---------------- tests/models/test_rules.py | 14 + 19 files changed, 1298 insertions(+), 1062 deletions(-) diff --git a/mex/common/models/__init__.py b/mex/common/models/__init__.py index 2116768c..9e92c983 100644 --- a/mex/common/models/__init__.py +++ b/mex/common/models/__init__.py @@ -30,8 +30,8 @@ contributing to specific fields of a merged item - `TRuleSet` classes are used for CRUD operations on a set of three rules -- `ExtractedTEntityFilter` defines how an entity filter specification should look like -- `ExtractedTMapping` defines how a raw data to extracted item mapping should look like +- `TFilter` defines how an entity filter specification should look like +- `TMapping` defines how a raw-data to extracted item mapping should look like Since these models for different use cases have a lot of overlapping attributes, we use a number of intermediate private classes to compose the public classes: @@ -63,8 +63,8 @@ - TRuleSetRequest: bundle of all three rules for one type used to create new rules - TRuleSetResponse: bundle of all three rules for one type including a `stableTargetId` -- ExtractedTEntityFilter: all BaseT fields re-typed as a list of entity filters -- ExtractedTMapping: all BaseT fields re-typed as lists of mapping fields +- TFilter: all BaseT fields re-typed as a list of filter field definitions +- TMapping: all BaseT fields re-typed as lists of mapping fields with `setValues` type In addition to the classes themselves, `mex.common.models` also exposes various lists of models, lookups by class name and typing for unions of models. @@ -73,6 +73,8 @@ from typing import Final, get_args from mex.common.models.access_platform import ( + AccessPlatformFilter, + AccessPlatformMapping, AccessPlatformRuleSetRequest, AccessPlatformRuleSetResponse, AdditiveAccessPlatform, @@ -84,6 +86,8 @@ SubtractiveAccessPlatform, ) from mex.common.models.activity import ( + ActivityFilter, + ActivityMapping, ActivityRuleSetRequest, ActivityRuleSetResponse, AdditiveActivity, @@ -95,14 +99,16 @@ SubtractiveActivity, ) from mex.common.models.base.extracted_data import ExtractedData -from mex.common.models.base.filter import generate_entity_filter_schema -from mex.common.models.base.mapping import generate_mapping_schema +from mex.common.models.base.filter import BaseFilter, FilterField, FilterRule +from mex.common.models.base.mapping import BaseMapping, MappingField, MappingRule from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.rules import AdditiveRule, PreventiveRule, SubtractiveRule from mex.common.models.bibliographic_resource import ( AdditiveBibliographicResource, BaseBibliographicResource, + BibliographicResourceFilter, + BibliographicResourceMapping, BibliographicResourceRuleSetRequest, BibliographicResourceRuleSetResponse, ExtractedBibliographicResource, @@ -114,6 +120,8 @@ from mex.common.models.consent import ( AdditiveConsent, BaseConsent, + ConsentFilter, + ConsentMapping, ConsentRuleSetRequest, ConsentRuleSetResponse, ExtractedConsent, @@ -125,6 +133,8 @@ from mex.common.models.contact_point import ( AdditiveContactPoint, BaseContactPoint, + ContactPointFilter, + ContactPointMapping, ContactPointRuleSetRequest, ContactPointRuleSetResponse, ExtractedContactPoint, @@ -136,6 +146,8 @@ from mex.common.models.distribution import ( AdditiveDistribution, BaseDistribution, + DistributionFilter, + DistributionMapping, DistributionRuleSetRequest, DistributionRuleSetResponse, ExtractedDistribution, @@ -149,6 +161,8 @@ BaseOrganization, ExtractedOrganization, MergedOrganization, + OrganizationFilter, + OrganizationMapping, OrganizationRuleSetRequest, OrganizationRuleSetResponse, PreventiveOrganization, @@ -160,6 +174,8 @@ BaseOrganizationalUnit, ExtractedOrganizationalUnit, MergedOrganizationalUnit, + OrganizationalUnitFilter, + OrganizationalUnitMapping, OrganizationalUnitRuleSetRequest, OrganizationalUnitRuleSetResponse, PreventiveOrganizationalUnit, @@ -171,6 +187,8 @@ BasePerson, ExtractedPerson, MergedPerson, + PersonFilter, + PersonMapping, PersonRuleSetRequest, PersonRuleSetResponse, PreventivePerson, @@ -184,6 +202,8 @@ MergedPrimarySource, PreventivePrimarySource, PreviewPrimarySource, + PrimarySourceFilter, + PrimarySourceMapping, PrimarySourceRuleSetRequest, PrimarySourceRuleSetResponse, SubtractivePrimarySource, @@ -195,6 +215,8 @@ MergedResource, PreventiveResource, PreviewResource, + ResourceFilter, + ResourceMapping, ResourceRuleSetRequest, ResourceRuleSetResponse, SubtractiveResource, @@ -207,6 +229,8 @@ PreventiveVariable, PreviewVariable, SubtractiveVariable, + VariableFilter, + VariableMapping, VariableRuleSetRequest, VariableRuleSetResponse, ) @@ -218,6 +242,8 @@ PreventiveVariableGroup, PreviewVariableGroup, SubtractiveVariableGroup, + VariableGroupFilter, + VariableGroupMapping, VariableGroupRuleSetRequest, VariableGroupRuleSetResponse, ) @@ -250,8 +276,12 @@ "RULE_SET_RESPONSE_CLASSES_BY_NAME", "SUBTRACTIVE_MODEL_CLASSES", "SUBTRACTIVE_MODEL_CLASSES_BY_NAME", + "AccessPlatformFilter", + "AccessPlatformMapping", "AccessPlatformRuleSetRequest", "AccessPlatformRuleSetResponse", + "ActivityFilter", + "ActivityMapping", "ActivityRuleSetRequest", "ActivityRuleSetResponse", "AdditiveAccessPlatform", @@ -283,6 +313,8 @@ "BaseConsent", "BaseContactPoint", "BaseDistribution", + "BaseFilter", + "BaseMapping", "BaseModel", "BaseOrganization", "BaseOrganizationalUnit", @@ -291,10 +323,18 @@ "BaseResource", "BaseVariable", "BaseVariableGroup", + "BibliographicResourceFilter", + "BibliographicResourceMapping", + "ConsentFilter", + "ConsentMapping", "ConsentRuleSetRequest", "ConsentRuleSetResponse", + "ContactPointFilter", + "ContactPointMapping", "ContactPointRuleSetRequest", "ContactPointRuleSetResponse", + "DistributionFilter", + "DistributionMapping", "DistributionRuleSetRequest", "DistributionRuleSetResponse", "ExtractedAccessPlatform", @@ -311,6 +351,10 @@ "ExtractedResource", "ExtractedVariable", "ExtractedVariableGroup", + "FilterField", + "FilterRule", + "MappingField", + "MappingRule", "MergedAccessPlatform", "MergedActivity", "MergedBibliographicResource", @@ -325,10 +369,16 @@ "MergedResource", "MergedVariable", "MergedVariableGroup", + "OrganizationFilter", + "OrganizationMapping", "OrganizationRuleSetRequest", "OrganizationRuleSetResponse", + "OrganizationalUnitFilter", + "OrganizationalUnitMapping", "OrganizationalUnitRuleSetRequest", "OrganizationalUnitRuleSetResponse", + "PersonFilter", + "PersonMapping", "PersonRuleSetRequest", "PersonRuleSetResponse", "PreventiveAccessPlatform", @@ -358,8 +408,12 @@ "PreviewResource", "PreviewVariable", "PreviewVariableGroup", + "PrimarySourceFilter", + "PrimarySourceMapping", "PrimarySourceRuleSetRequest", "PrimarySourceRuleSetResponse", + "ResourceFilter", + "ResourceMapping", "ResourceRuleSetRequest", "ResourceRuleSetResponse", "SubtractiveAccessPlatform", @@ -376,8 +430,12 @@ "SubtractiveRule", "SubtractiveVariable", "SubtractiveVariableGroup", + "VariableFilter", + "VariableGroupFilter", + "VariableGroupMapping", "VariableGroupRuleSetRequest", "VariableGroupRuleSetResponse", + "VariableMapping", "VariableRuleSetRequest", "VariableRuleSetResponse", ) @@ -586,10 +644,52 @@ cls.__name__: cls for cls in RULE_SET_RESPONSE_CLASSES } -FILTER_MODEL_BY_EXTRACTED_CLASS_NAME = { - cls.__name__: generate_entity_filter_schema(cls) for cls in EXTRACTED_MODEL_CLASSES +AnyMappingModel = ( + AccessPlatformMapping + | ActivityMapping + | BibliographicResourceMapping + | ConsentMapping + | ContactPointMapping + | DistributionMapping + | OrganizationMapping + | OrganizationalUnitMapping + | PersonMapping + | PrimarySourceMapping + | ResourceMapping + | VariableMapping + | VariableGroupMapping +) +MAPPING_MODEL_CLASSES: Final[list[type[AnyMappingModel]]] = list( + get_args(AnyMappingModel) +) +MAPPING_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyMappingModel]]] = { + cls.__name__: cls for cls in MAPPING_MODEL_CLASSES } - +# MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME is deprecated, use MAPPING_MODEL_CLASSES_BY_NAME MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME = { - cls.__name__: generate_mapping_schema(cls) for cls in EXTRACTED_MODEL_CLASSES + f"Extracted{cls.stemType}": cls for cls in MAPPING_MODEL_CLASSES +} + +AnyFilterModel = ( + AccessPlatformFilter + | ActivityFilter + | BibliographicResourceFilter + | ConsentFilter + | ContactPointFilter + | DistributionFilter + | OrganizationFilter + | OrganizationalUnitFilter + | PersonFilter + | PrimarySourceFilter + | ResourceFilter + | VariableFilter + | VariableGroupFilter +) +FILTER_MODEL_CLASSES: Final[list[type[AnyFilterModel]]] = list(get_args(AnyFilterModel)) +FILTER_MODEL_CLASSES_BY_NAME: Final[dict[str, type[AnyFilterModel]]] = { + cls.__name__: cls for cls in FILTER_MODEL_CLASSES +} +# FILTER_MODEL_BY_EXTRACTED_CLASS_NAME is deprecated, use FILTER_MODEL_CLASSES_BY_NAME +FILTER_MODEL_BY_EXTRACTED_CLASS_NAME = { + f"Extracted{cls.stemType}": cls for cls in FILTER_MODEL_CLASSES } diff --git a/mex/common/models/access_platform.py b/mex/common/models/access_platform.py index 54426e9f..137dee0d 100644 --- a/mex/common/models/access_platform.py +++ b/mex/common/models/access_platform.py @@ -5,6 +5,8 @@ from pydantic import AfterValidator, Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -28,6 +30,13 @@ Text, ) +AnyContactIdentifier = Annotated[ + MergedOrganizationalUnitIdentifier + | MergedPersonIdentifier + | MergedContactPointIdentifier, + AfterValidator(Identifier), +] + class _Stem(BaseModel): stemType: ClassVar[Annotated[Literal["AccessPlatform"], Field(frozen=True)]] = ( @@ -37,14 +46,7 @@ class _Stem(BaseModel): class _OptionalLists(_Stem): alternativeTitle: list[Text] = [] - contact: list[ - Annotated[ - MergedOrganizationalUnitIdentifier - | MergedPersonIdentifier - | MergedContactPointIdentifier, - AfterValidator(Identifier), - ] - ] = [] + contact: list[AnyContactIdentifier] = [] description: list[Text] = [] landingPage: list[Link] = [] title: list[Text] = [] @@ -173,3 +175,47 @@ class AccessPlatformRuleSetResponse(_BaseRuleSet): Literal["AccessPlatformRuleSetResponse"], Field(alias="$type", frozen=True) ] = "AccessPlatformRuleSetResponse" stableTargetId: MergedAccessPlatformIdentifier + + +class AccessPlatformMapping(_Stem, BaseMapping): + """Mapping for describing an access platform transformation.""" + + entityType: Annotated[ + Literal["AccessPlatformMapping"], Field(alias="$type", frozen=True) + ] = "AccessPlatformMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + technicalAccessibility: Annotated[ + list[MappingField[TechnicalAccessibility]], Field(min_length=1) + ] + endpointDescription: list[MappingField[Link | None]] = [] + endpointType: list[MappingField[APIType | None]] = [] + endpointURL: list[MappingField[Link | None]] = [] + alternativeTitle: list[MappingField[list[Text]]] = [] + contact: list[MappingField[list[AnyContactIdentifier]]] = [] + description: list[MappingField[list[Text]]] = [] + landingPage: list[MappingField[list[Link]]] = [] + title: list[MappingField[list[Text]]] = [] + unitInCharge: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = [] + + +class AccessPlatformFilter(_Stem, BaseFilter): + """Class for defining filter rules for access platform items.""" + + entityType: Annotated[ + Literal["AccessPlatformFilter"], Field(alias="$type", frozen=True) + ] = "AccessPlatformFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + alternativeTitle: list[FilterField] = [] + contact: list[FilterField] = [] + description: list[FilterField] = [] + endpointDescription: list[FilterField] = [] + endpointType: list[FilterField] = [] + endpointURL: list[FilterField] = [] + landingPage: list[FilterField] = [] + technicalAccessibility: list[FilterField] = [] + title: list[FilterField] = [] + unitInCharge: list[FilterField] = [] diff --git a/mex/common/models/activity.py b/mex/common/models/activity.py index 0258ca60..79013ee5 100644 --- a/mex/common/models/activity.py +++ b/mex/common/models/activity.py @@ -8,6 +8,8 @@ from pydantic import AfterValidator, Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -36,6 +38,17 @@ YearMonthDay, ) +AnyExternalAssociateIdentifier = Annotated[ + MergedOrganizationIdentifier | MergedPersonIdentifier, + AfterValidator(Identifier), +] +AnyContactIdentifier = Annotated[ + MergedOrganizationalUnitIdentifier + | MergedPersonIdentifier + | MergedContactPointIdentifier, + AfterValidator(Identifier), +] + class _Stem(BaseModel): stemType: ClassVar[Annotated[Literal["Activity"], Field(frozen=True)]] = "Activity" @@ -47,12 +60,7 @@ class _OptionalLists(_Stem): alternativeTitle: list[Text] = [] documentation: list[Link] = [] end: list[YearMonthDay | YearMonth | Year] = [] - externalAssociate: list[ - Annotated[ - MergedOrganizationIdentifier | MergedPersonIdentifier, - AfterValidator(Identifier), - ] - ] = [] + externalAssociate: list[AnyExternalAssociateIdentifier] = [] funderOrCommissioner: list[MergedOrganizationIdentifier] = [] fundingProgram: list[str] = [] involvedPerson: list[MergedPersonIdentifier] = [] @@ -67,17 +75,7 @@ class _OptionalLists(_Stem): class _RequiredLists(_Stem): - contact: Annotated[ - list[ - Annotated[ - MergedOrganizationalUnitIdentifier - | MergedPersonIdentifier - | MergedContactPointIdentifier, - AfterValidator(Identifier), - ] - ], - Field(min_length=1), - ] + contact: Annotated[list[AnyContactIdentifier], Field(min_length=1)] responsibleUnit: Annotated[ list[MergedOrganizationalUnitIdentifier], Field(min_length=1) ] @@ -85,14 +83,7 @@ class _RequiredLists(_Stem): class _SparseLists(_Stem): - contact: list[ - Annotated[ - MergedOrganizationalUnitIdentifier - | MergedPersonIdentifier - | MergedContactPointIdentifier, - AfterValidator(Identifier), - ] - ] = [] + contact: list[AnyContactIdentifier] = [] responsibleUnit: list[MergedOrganizationalUnitIdentifier] = [] title: list[Text] = [] @@ -204,3 +195,70 @@ class ActivityRuleSetResponse(_BaseRuleSet): Literal["ActivityRuleSetResponse"], Field(alias="$type", frozen=True) ] = "ActivityRuleSetResponse" stableTargetId: MergedActivityIdentifier + + +class ActivityMapping(_Stem, BaseMapping): + """Mapping for describing a activity transformation.""" + + entityType: Annotated[ + Literal["ActivityMapping"], Field(alias="$type", frozen=True) + ] = "ActivityMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + contact: Annotated[ + list[MappingField[list[AnyContactIdentifier]]], Field(min_length=1) + ] + responsibleUnit: Annotated[ + list[MappingField[list[MergedOrganizationalUnitIdentifier]]], + Field(min_length=1), + ] + title: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + abstract: list[MappingField[list[Text]]] = [] + activityType: list[MappingField[list[ActivityType]]] = [] + alternativeTitle: list[MappingField[list[Text]]] = [] + documentation: list[MappingField[list[Link]]] = [] + end: list[MappingField[list[YearMonthDay | YearMonth | Year]]] = [] + externalAssociate: list[MappingField[list[AnyExternalAssociateIdentifier]]] = [] + funderOrCommissioner: list[MappingField[list[MergedOrganizationIdentifier]]] = [] + fundingProgram: list[MappingField[list[str]]] = [] + involvedPerson: list[MappingField[list[MergedPersonIdentifier]]] = [] + involvedUnit: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = [] + isPartOfActivity: list[MappingField[list[MergedActivityIdentifier]]] = [] + publication: list[MappingField[list[MergedBibliographicResourceIdentifier]]] = [] + shortName: list[MappingField[list[Text]]] = [] + start: list[MappingField[list[YearMonthDay | YearMonth | Year]]] = [] + succeeds: list[MappingField[list[MergedActivityIdentifier]]] = [] + theme: list[MappingField[list[Theme]]] = [] + website: list[MappingField[list[Link]]] = [] + + +class ActivityFilter(_Stem, BaseFilter): + """Class for defining filter rules for activity items.""" + + entityType: Annotated[ + Literal["ActivityFilter"], Field(alias="$type", frozen=True) + ] = "ActivityFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + abstract: list[FilterField] = [] + activityType: list[FilterField] = [] + alternativeTitle: list[FilterField] = [] + contact: list[FilterField] = [] + documentation: list[FilterField] = [] + end: list[FilterField] = [] + externalAssociate: list[FilterField] = [] + funderOrCommissioner: list[FilterField] = [] + fundingProgram: list[FilterField] = [] + involvedPerson: list[FilterField] = [] + involvedUnit: list[FilterField] = [] + isPartOfActivity: list[FilterField] = [] + publication: list[FilterField] = [] + responsibleUnit: list[FilterField] = [] + shortName: list[FilterField] = [] + start: list[FilterField] = [] + succeeds: list[FilterField] = [] + theme: list[FilterField] = [] + title: list[FilterField] = [] + website: list[FilterField] = [] diff --git a/mex/common/models/base/filter.py b/mex/common/models/base/filter.py index 3387f5de..c62b0f9a 100644 --- a/mex/common/models/base/filter.py +++ b/mex/common/models/base/filter.py @@ -1,52 +1,30 @@ -from typing import TYPE_CHECKING, Annotated, Any +from typing import Annotated -from pydantic import BaseModel, Field, create_model +from pydantic import BaseModel, Field -from mex.common.transform import ensure_postfix -if TYPE_CHECKING: # pragma: no cover - from mex.common.models import AnyExtractedModel - - -class EntityFilterRule(BaseModel, extra="forbid"): +class FilterRule(BaseModel, extra="forbid"): """Entity filter rule model.""" - forValues: list[str] | None = None - rule: str | None = None - - -class EntityFilter(BaseModel, extra="forbid"): - """Entity filter model.""" + forValues: Annotated[list[str] | None, Field(title="forValues")] = None + rule: Annotated[str | None, Field(title="rule")] = None - fieldInPrimarySource: str - locationInPrimarySource: str | None = None - examplesInPrimarySource: list[str] | None = None - mappingRules: Annotated[list[EntityFilterRule], Field(min_length=1)] - comment: str | None = None +class FilterField(BaseModel, extra="forbid"): + """Entity filter field model.""" -def generate_entity_filter_schema( - extracted_model: type["AnyExtractedModel"], -) -> type[BaseModel]: - """Create a mapping schema for an entity filter for an extracted model class. - - Example entity filter: If activity starts before 2016: do not extract. - - Args: - extracted_model: a pydantic model for an extracted model class - - Returns: - model of the mapping schema for an entity filter - """ - fields: dict[str, Any] = { - extracted_model.__name__: (list[EntityFilter], None), - } - entity_filter_name = ensure_postfix(extracted_model.stemType, "EntityFilter") - entity_filter_model: type[BaseModel] = create_model( - entity_filter_name, - **fields, - ) - entity_filter_model.__doc__ = ( - f"Schema for entity filters for the entity type {extracted_model.__name__}." + fieldInPrimarySource: Annotated[str | None, Field(title="fieldInPrimarySource")] = ( + None ) - return entity_filter_model + locationInPrimarySource: Annotated[ + str | None, Field(title="locationInPrimarySource") + ] = None + examplesInPrimarySource: Annotated[ + list[str] | None, Field(title="examplesInPrimarySource") + ] = None + mappingRules: Annotated[list[FilterRule], Field(min_length=1, title="mappingRules")] + comment: Annotated[str | None, Field(title="comment")] = None + + +class BaseFilter(BaseModel, extra="forbid"): + """Base class for filter implementations.""" diff --git a/mex/common/models/base/mapping.py b/mex/common/models/base/mapping.py index 35788116..f91a3a7d 100644 --- a/mex/common/models/base/mapping.py +++ b/mex/common/models/base/mapping.py @@ -1,25 +1,24 @@ -from typing import TYPE_CHECKING, Annotated, Any, get_origin +from typing import Annotated, Generic, TypeVar -from pydantic import BaseModel, Field, create_model +from pydantic import BaseModel, Field -from mex.common.transform import ensure_postfix +T = TypeVar("T") -if TYPE_CHECKING: # pragma: no cover - from mex.common.models import AnyExtractedModel - -class BaseMappingRule(BaseModel, extra="forbid"): +class MappingRule(BaseModel, Generic[T], extra="forbid"): """Generic mapping rule model.""" forValues: Annotated[list[str] | None, Field(title="forValues")] = None - setValues: Annotated[list[Any] | None, Field(title="setValues")] = None + setValues: Annotated[list[T] | None, Field(title="setValues")] = None rule: Annotated[str | None, Field(title="rule")] = None -class BaseMappingField(BaseModel, extra="forbid"): +class MappingField(BaseModel, Generic[T], extra="forbid"): """Generic mapping field model.""" - fieldInPrimarySource: Annotated[str, Field(title="fieldInPrimarySource")] + fieldInPrimarySource: Annotated[str | None, Field(title="fieldInPrimarySource")] = ( + None + ) locationInPrimarySource: Annotated[ str | None, Field(title="locationInPrimarySource") ] = None @@ -27,118 +26,10 @@ class BaseMappingField(BaseModel, extra="forbid"): list[str] | None, Field(title="examplesInPrimarySource") ] = None mappingRules: Annotated[ - list[BaseMappingRule], Field(min_length=1, title="mappingRules") + list[MappingRule[T]], Field(min_length=1, title="mappingRules") ] comment: Annotated[str | None, Field(title="comment")] = None -def generate_mapping_schema( - extracted_model: type["AnyExtractedModel"], -) -> type[BaseModel]: - """Create a mapping schema the MEx extracted model class. - - Pydantic models are dynamically created for the given entity type from - depending on the respective fields and their types. - - Args: - extracted_model: a pydantic model for an extracted model class - - Returns: - dynamic mapping model for the provided extracted model class - """ - # dicts for create_model() must be declared as dict[str, Any] to silence mypy - fields: dict[str, Any] = {} - for field_name, field_info in extracted_model.model_fields.items(): - if field_name == "entityType": - continue - # first create dynamic rule model - if get_origin(field_info.annotation) is list: - rule_type: Any = field_info.annotation - else: - rule_type = list[field_info.annotation] # type: ignore[name-defined] - - field_class_name = field_name[0].upper() + field_name[1:] - - rule_model: type[BaseMappingRule] = create_model( - f"{field_class_name}MappingRule", - __base__=(BaseMappingRule,), - setValues=( - Annotated[rule_type | None, Field(title="setValues")], - None, - ), - ) - rule_model.__doc__ = str(f"Mapping rule schema for field {field_name}.") - # then update the mappingRules type in the field in primary source schema - field_model: type[BaseMappingField] = create_model( - f"{field_class_name}MappingField", - __base__=(BaseMappingField,), - mappingRules=( - list[rule_model], # type: ignore[valid-type] - Field(..., min_length=1, title="mappingRules"), - ), - ) - field_model.__doc__ = str( - f"Mapping schema for {field_name} fields in primary source." - ) - if field_info.is_required(): - fields[field_name] = ( - Annotated[list[field_model], Field(title=field_name)], # type: ignore[valid-type] - ..., - ) - else: - fields[field_name] = ( - Annotated[list[field_model], Field(title=field_name)], # type: ignore[valid-type] - [], - ) - mapping_name = ensure_postfix(extracted_model.stemType, "Mapping") - mapping_model: type[BaseModel] = create_model(mapping_name, **fields) - mapping_model.__doc__ = ( - "Schema for mapping the properties of the entity type " - f"{extracted_model.__name__}." - ) - return mapping_model - - -def _materialize_mapping_schemas() -> None: - import json - from pathlib import Path - from subprocess import run - - from mex.common.models import MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME - - out_dir = Path.cwd() - - for name, model in MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME.items(): - with open( - out_dir / f"{name}_MappingSchema.json", - "w", - encoding="utf-8", - ) as fh: - fh.write( - json.dumps( - model.model_json_schema(), - ensure_ascii=False, - indent=4, - ) - + "\n" - ) - print("written", out_dir / f"{name}_MappingSchema.json") # noqa: T201 - run( # noqa: S603 - [ # noqa: S607 - "datamodel-codegen", - "--input", - out_dir / f"{name}_MappingSchema.json", - "--input-file-type", - "jsonschema", - "--output", - out_dir / f"{name}.py", - ], - check=False, - ) - print("written", out_dir / f"{name}.py") # noqa: T201 - - break - - -if __name__ == "__main__": - _materialize_mapping_schemas() +class BaseMapping(BaseModel, extra="forbid"): + """Base class for mapping implementations.""" diff --git a/mex/common/models/bibliographic_resource.py b/mex/common/models/bibliographic_resource.py index a760e379..e6734bbe 100644 --- a/mex/common/models/bibliographic_resource.py +++ b/mex/common/models/bibliographic_resource.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -75,6 +77,7 @@ ], ), ] +PagesStr = Annotated[str, Field(examples=["1", "45-67", "45 - 67", "II", "XI", "10i"])] PublicationPlaceStr = Annotated[ str, Field( @@ -153,10 +156,7 @@ class _OptionalValues(_Stem): issue: VolumeOrIssueStr | None = None issued: YearMonthDayTime | YearMonthDay | YearMonth | Year | None = None license: License | None = None - pages: ( - Annotated[str, Field(examples=["1", "45-67", "45 - 67", "II", "XI", "10i"])] - | None - ) = None + pages: PagesStr | None = None publicationPlace: PublicationPlaceStr | None = None publicationYear: Year | None = None repositoryURL: Link | None = None @@ -180,9 +180,7 @@ class _VariadicValues(_Stem): issue: list[VolumeOrIssueStr] = [] issued: list[YearMonthDayTime | YearMonthDay | YearMonth | Year] = [] license: list[License] = [] - pages: list[ - Annotated[str, Field(examples=["1", "45-67", "45 - 67", "II", "XI", "10i"])] - ] = [] + pages: list[PagesStr] = [] publicationPlace: list[PublicationPlaceStr] = [] publicationYear: list[Year] = [] repositoryURL: list[Link] = [] @@ -319,3 +317,93 @@ class BibliographicResourceRuleSetResponse(_BaseRuleSet): Field(alias="$type", frozen=True), ] = "BibliographicResourceRuleSetResponse" stableTargetId: MergedBibliographicResourceIdentifier + + +class BibliographicResourceMapping(_Stem, BaseMapping): + """Mapping for describing a bibliographic resource transformation.""" + + entityType: Annotated[ + Literal["BibliographicResourceMapping"], Field(alias="$type", frozen=True) + ] = "BibliographicResourceMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + accessRestriction: Annotated[ + list[MappingField[AccessRestriction]], Field(min_length=1) + ] + doi: list[MappingField[DoiStr | None]] = [] + edition: list[MappingField[EditionStr | None]] = [] + issue: list[MappingField[VolumeOrIssueStr | None]] = [] + issued: list[ + MappingField[YearMonthDayTime | YearMonthDay | YearMonth | Year | None] + ] = [] + license: list[MappingField[License | None]] = [] + pages: list[MappingField[PagesStr | None]] = [] + publicationPlace: list[MappingField[PublicationPlaceStr | None]] = [] + publicationYear: list[MappingField[Year | None]] = [] + repositoryURL: list[MappingField[Link | None]] = [] + section: list[MappingField[SectionStr | None]] = [] + volume: list[MappingField[VolumeOrIssueStr | None]] = [] + volumeOfSeries: list[MappingField[VolumeOrIssueStr | None]] = [] + creator: Annotated[ + list[MappingField[list[MergedPersonIdentifier]]], Field(min_length=1) + ] + title: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + abstract: list[MappingField[list[Text]]] = [] + alternateIdentifier: list[MappingField[list[str]]] = [] + alternativeTitle: list[MappingField[list[Text]]] = [] + bibliographicResourceType: list[MappingField[list[BibliographicResourceType]]] = [] + contributingUnit: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = [] + distribution: list[MappingField[list[MergedDistributionIdentifier]]] = [] + editor: list[MappingField[list[MergedPersonIdentifier]]] = [] + editorOfSeries: list[MappingField[list[MergedPersonIdentifier]]] = [] + isbnIssn: list[MappingField[list[IsbnIssnStr]]] = [] + journal: list[MappingField[list[Text]]] = [] + keyword: list[MappingField[list[Text]]] = [] + language: list[MappingField[list[Language]]] = [] + publisher: list[MappingField[list[MergedOrganizationIdentifier]]] = [] + subtitle: list[MappingField[list[Text]]] = [] + titleOfBook: list[MappingField[list[Text]]] = [] + titleOfSeries: list[MappingField[list[Text]]] = [] + + +class BibliographicResourceFilter(_Stem, BaseFilter): + """Class for defining filter rules for bibliographic resource items.""" + + entityType: Annotated[ + Literal["BibliographicResourceFilter"], Field(alias="$type", frozen=True) + ] = "BibliographicResourceFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + abstract: list[FilterField] = [] + accessRestriction: list[FilterField] = [] + alternateIdentifier: list[FilterField] = [] + alternativeTitle: list[FilterField] = [] + bibliographicResourceType: list[FilterField] = [] + contributingUnit: list[FilterField] = [] + creator: list[FilterField] = [] + distribution: list[FilterField] = [] + doi: list[FilterField] = [] + edition: list[FilterField] = [] + editor: list[FilterField] = [] + editorOfSeries: list[FilterField] = [] + isbnIssn: list[FilterField] = [] + issue: list[FilterField] = [] + issued: list[FilterField] = [] + journal: list[FilterField] = [] + keyword: list[FilterField] = [] + language: list[FilterField] = [] + license: list[FilterField] = [] + pages: list[FilterField] = [] + publicationPlace: list[FilterField] = [] + publicationYear: list[FilterField] = [] + publisher: list[FilterField] = [] + repositoryURL: list[FilterField] = [] + section: list[FilterField] = [] + subtitle: list[FilterField] = [] + title: list[FilterField] = [] + titleOfBook: list[FilterField] = [] + titleOfSeries: list[FilterField] = [] + volume: list[FilterField] = [] + volumeOfSeries: list[FilterField] = [] diff --git a/mex/common/models/consent.py b/mex/common/models/consent.py index de224782..a71c25bf 100644 --- a/mex/common/models/consent.py +++ b/mex/common/models/consent.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -143,3 +145,37 @@ class ConsentRuleSetResponse(_BaseRuleSet): Literal["ConsentRuleSetResponse"], Field(alias="$type", frozen=True) ] = "ConsentRuleSetResponse" stableTargetId: MergedConsentIdentifier + + +class ConsentMapping(_Stem, BaseMapping): + """Mapping for describing a consent transformation.""" + + entityType: Annotated[ + Literal["ConsentMapping"], Field(alias="$type", frozen=True) + ] = "ConsentMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + hasConsentStatus: Annotated[list[MappingField[ConsentStatus]], Field(min_length=1)] + hasDataSubject: Annotated[ + list[MappingField[MergedPersonIdentifier]], Field(min_length=1) + ] + isIndicatedAtTime: Annotated[ + list[MappingField[YearMonthDayTime]], Field(min_length=1) + ] + hasConsentType: list[MappingField[ConsentType | None]] = [] + + +class ConsentFilter(_Stem, BaseFilter): + """Class for defining filter rules for consent items.""" + + entityType: Annotated[ + Literal["ConsentFilter"], Field(alias="$type", frozen=True) + ] = "ConsentFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + hasConsentType: list[FilterField] = [] + hasConsentStatus: list[FilterField] = [] + hasDataSubject: list[FilterField] = [] + isIndicatedAtTime: list[FilterField] = [] diff --git a/mex/common/models/contact_point.py b/mex/common/models/contact_point.py index a788e7ff..939ee0d4 100644 --- a/mex/common/models/contact_point.py +++ b/mex/common/models/contact_point.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -124,3 +126,27 @@ class ContactPointRuleSetResponse(_BaseRuleSet): Literal["ContactPointRuleSetResponse"], Field(alias="$type", frozen=True) ] = "ContactPointRuleSetResponse" stableTargetId: MergedContactPointIdentifier + + +class ContactPointMapping(_Stem, BaseMapping): + """Mapping for describing a contact point transformation.""" + + entityType: Annotated[ + Literal["ContactPointMapping"], Field(alias="$type", frozen=True) + ] = "ContactPointMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + email: Annotated[list[MappingField[list[Email]]], Field(min_length=1)] + + +class ContactPointFilter(_Stem, BaseFilter): + """Class for defining filter rules for contact point items.""" + + entityType: Annotated[ + Literal["ContactPointFilter"], Field(alias="$type", frozen=True) + ] = "ContactPointFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + email: list[FilterField] = [] diff --git a/mex/common/models/distribution.py b/mex/common/models/distribution.py index 269a7e17..b585c6ee 100644 --- a/mex/common/models/distribution.py +++ b/mex/common/models/distribution.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -180,3 +182,50 @@ class DistributionRuleSetResponse(_BaseRuleSet): Literal["DistributionRuleSetRequest"], Field(alias="$type", frozen=True) ] = "DistributionRuleSetRequest" stableTargetId: MergedAccessPlatformIdentifier + + +class DistributionMapping(_Stem, BaseMapping): + """Mapping for describing a distribution transformation.""" + + entityType: Annotated[ + Literal["DistributionMapping"], Field(alias="$type", frozen=True) + ] = "DistributionMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + accessRestriction: Annotated[ + list[MappingField[AccessRestriction]], Field(min_length=1) + ] + issued: Annotated[ + list[MappingField[YearMonthDayTime | YearMonthDay | YearMonth | Year]], + Field(min_length=1), + ] + accessService: list[MappingField[MergedAccessPlatformIdentifier | None]] = [] + license: list[MappingField[License | None]] = [] + mediaType: list[MappingField[MIMEType | None]] = [] + modified: list[ + MappingField[YearMonthDayTime | YearMonthDay | YearMonth | Year | None] + ] = [] + title: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + accessURL: list[MappingField[list[Link]]] = [] + downloadURL: list[MappingField[list[Link]]] = [] + + +class DistributionFilter(_Stem, BaseFilter): + """Class for defining filter rules for distribution items.""" + + entityType: Annotated[ + Literal["DistributionFilter"], Field(alias="$type", frozen=True) + ] = "DistributionFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + accessRestriction: list[FilterField] = [] + accessService: list[FilterField] = [] + accessURL: list[FilterField] = [] + downloadURL: list[FilterField] = [] + issued: list[FilterField] = [] + license: list[FilterField] = [] + mediaType: list[FilterField] = [] + modified: list[FilterField] = [] + title: list[FilterField] = [] diff --git a/mex/common/models/organization.py b/mex/common/models/organization.py index c1e2fe0f..2ac0380a 100644 --- a/mex/common/models/organization.py +++ b/mex/common/models/organization.py @@ -8,6 +8,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -24,6 +26,55 @@ Text, ) +GeprisIdStr = Annotated[ + str, + Field( + pattern=r"^https://gepris\.dfg\.de/gepris/institution/[0-9]{1,64}$", + examples=["https://gepris.dfg.de/gepris/institution/10179"], + json_schema_extra={"format": "uri"}, + ), +] +GndIdStr = Annotated[ + str, + Field( + pattern=r"^https://d\-nb\.info/gnd/[-X0-9]{3,10}$", + examples=["https://d-nb.info/gnd/17690-4"], + json_schema_extra={"format": "uri"}, + ), +] +IsniIdStr = Annotated[ + str, + Field( + pattern=r"^https://isni\.org/isni/[X0-9]{16}$", + examples=["https://isni.org/isni/0000000109403744"], + json_schema_extra={"format": "uri"}, + ), +] +RorIdStr = Annotated[ + str, + Field( + pattern=r"^https://ror\.org/[a-z0-9]{9}$", + examples=["https://ror.org/01k5qnb77"], + json_schema_extra={"format": "uri"}, + ), +] +ViafIdStr = Annotated[ + str, + Field( + pattern=r"^https://viaf\.org/viaf/[0-9]{2,22}$", + examples=["https://viaf.org/viaf/123556639"], + json_schema_extra={"format": "uri"}, + ), +] +WikidataIdStr = Annotated[ + str, + Field( + examples=["http://www.wikidata.org/entity/Q679041"], + pattern=r"^https://www\.wikidata\.org/entity/[PQ0-9]{2,64}$", + json_schema_extra={"format": "uri"}, + ), +] + class _Stem(BaseModel): stemType: ClassVar[Annotated[Literal["Organization"], Field(frozen=True)]] = ( @@ -33,67 +84,13 @@ class _Stem(BaseModel): class _OptionalLists(_Stem): alternativeName: list[Text] = [] - geprisId: list[ - Annotated[ - str, - Field( - pattern=r"^https://gepris\.dfg\.de/gepris/institution/[0-9]{1,64}$", - examples=["https://gepris.dfg.de/gepris/institution/10179"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] - gndId: list[ - Annotated[ - str, - Field( - pattern=r"^https://d\-nb\.info/gnd/[-X0-9]{3,10}$", - examples=["https://d-nb.info/gnd/17690-4"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] - isniId: list[ - Annotated[ - str, - Field( - pattern=r"^https://isni\.org/isni/[X0-9]{16}$", - examples=["https://isni.org/isni/0000000109403744"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] - rorId: list[ - Annotated[ - str, - Field( - pattern=r"^https://ror\.org/[a-z0-9]{9}$", - examples=["https://ror.org/01k5qnb77"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] + geprisId: list[GeprisIdStr] = [] + gndId: list[GndIdStr] = [] + isniId: list[IsniIdStr] = [] + rorId: list[RorIdStr] = [] shortName: list[Text] = [] - viafId: list[ - Annotated[ - str, - Field( - pattern=r"^https://viaf\.org/viaf/[0-9]{2,22}$", - examples=["https://viaf.org/viaf/123556639"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] - wikidataId: list[ - Annotated[ - str, - Field( - examples=["http://www.wikidata.org/entity/Q679041"], - pattern=r"^https://www\.wikidata\.org/entity/[PQ0-9]{2,64}$", - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] + viafId: list[ViafIdStr] = [] + wikidataId: list[WikidataIdStr] = [] class _RequiredLists(_Stem): @@ -200,3 +197,43 @@ class OrganizationRuleSetResponse(_BaseRuleSet): Literal["OrganizationRuleSetResponse"], Field(alias="$type", frozen=True) ] = "OrganizationRuleSetResponse" stableTargetId: MergedOrganizationIdentifier + + +class OrganizationMapping(_Stem, BaseMapping): + """Mapping for describing a organization transformation.""" + + entityType: Annotated[ + Literal["OrganizationMapping"], Field(alias="$type", frozen=True) + ] = "OrganizationMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + officialName: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + alternativeName: list[MappingField[list[Text]]] = [] + geprisId: list[MappingField[list[GeprisIdStr]]] = [] + gndId: list[MappingField[list[GndIdStr]]] = [] + isniId: list[MappingField[list[IsniIdStr]]] = [] + rorId: list[MappingField[list[RorIdStr]]] = [] + shortName: list[MappingField[list[Text]]] = [] + viafId: list[MappingField[list[ViafIdStr]]] = [] + wikidataId: list[MappingField[list[WikidataIdStr]]] = [] + + +class OrganizationFilter(_Stem, BaseFilter): + """Class for defining filter rules for organization items.""" + + entityType: Annotated[ + Literal["OrganizationFilter"], Field(alias="$type", frozen=True) + ] = "OrganizationFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + alternativeName: list[FilterField] = [] + geprisId: list[FilterField] = [] + gndId: list[FilterField] = [] + isniId: list[FilterField] = [] + officialName: list[FilterField] = [] + rorId: list[FilterField] = [] + shortName: list[FilterField] = [] + viafId: list[FilterField] = [] + wikidataId: list[FilterField] = [] diff --git a/mex/common/models/organizational_unit.py b/mex/common/models/organizational_unit.py index d89a15ef..6f3ff04c 100644 --- a/mex/common/models/organizational_unit.py +++ b/mex/common/models/organizational_unit.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -155,3 +157,39 @@ class OrganizationalUnitRuleSetResponse(_BaseRuleSet): Literal["OrganizationalUnitRuleSetResponse"], Field(alias="$type", frozen=True) ] = "OrganizationalUnitRuleSetResponse" stableTargetId: MergedOrganizationalUnitIdentifier + + +class OrganizationalUnitMapping(_Stem, BaseMapping): + """Mapping for describing an organizational unit transformation.""" + + entityType: Annotated[ + Literal["OrganizationalUnitMapping"], Field(alias="$type", frozen=True) + ] = "OrganizationalUnitMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + parentUnit: list[MappingField[MergedOrganizationalUnitIdentifier | None]] = [] + name: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + alternativeName: list[MappingField[list[Text]]] = [] + email: list[MappingField[list[Email]]] = [] + shortName: list[MappingField[list[Text]]] = [] + unitOf: list[MappingField[list[MergedOrganizationIdentifier]]] = [] + website: list[MappingField[list[Link]]] = [] + + +class OrganizationalUnitFilter(_Stem, BaseFilter): + """Class for defining filter rules for organizational unit items.""" + + entityType: Annotated[ + Literal["OrganizationalUnitFilter"], Field(alias="$type", frozen=True) + ] = "OrganizationalUnitFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + alternativeName: list[FilterField] = [] + email: list[FilterField] = [] + name: list[FilterField] = [] + parentUnit: list[FilterField] = [] + shortName: list[FilterField] = [] + unitOf: list[FilterField] = [] + website: list[FilterField] = [] diff --git a/mex/common/models/person.py b/mex/common/models/person.py index 482d8941..8f0507ff 100644 --- a/mex/common/models/person.py +++ b/mex/common/models/person.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -23,6 +25,41 @@ MergedPrimarySourceIdentifier, ) +FamilyNameStr = Annotated[ + str, + Field( + examples=["Patapoutian", "Skłodowska-Curie", "Muta Maathai"], + ), +] +FullNameStr = Annotated[ + str, + Field( + examples=["Juturna Felicitás", "M. Berhanu", "Keone Seong-Hyeon"], + ), +] +GivenNameStr = Annotated[ + str, + Field( + examples=["Romāns", "Marie Salomea", "May-Britt"], + ), +] +IsniIdStr = Annotated[ + str, + Field( + pattern=r"^https://isni\.org/isni/[X0-9]{16}$", + examples=["https://isni.org/isni/0000000109403744"], + json_schema_extra={"format": "uri"}, + ), +] +OrcidIdStr = Annotated[ + str, + Field( + pattern=r"^https://orcid\.org/[-X0-9]{9,21}$", + examples=["https://orcid.org/0000-0002-9079-593X"], + json_schema_extra={"format": "uri"}, + ), +] + class _Stem(BaseModel): stemType: ClassVar[Annotated[Literal["Person"], Field(frozen=True)]] = "Person" @@ -31,51 +68,12 @@ class _Stem(BaseModel): class _OptionalLists(_Stem): affiliation: list[MergedOrganizationIdentifier] = [] email: list[Email] = [] - familyName: list[ - Annotated[ - str, - Field( - examples=["Patapoutian", "Skłodowska-Curie", "Muta Maathai"], - ), - ] - ] = [] - fullName: list[ - Annotated[ - str, - Field( - examples=["Juturna Felicitás", "M. Berhanu", "Keone Seong-Hyeon"], - ), - ] - ] = [] - givenName: list[ - Annotated[ - str, - Field( - examples=["Romāns", "Marie Salomea", "May-Britt"], - ), - ] - ] = [] - isniId: list[ - Annotated[ - str, - Field( - pattern=r"^https://isni\.org/isni/[X0-9]{16}$", - examples=["https://isni.org/isni/0000000109403744"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] + familyName: list[FamilyNameStr] = [] + fullName: list[FullNameStr] = [] + givenName: list[GivenNameStr] = [] + isniId: list[IsniIdStr] = [] memberOf: list[MergedOrganizationalUnitIdentifier] = [] - orcidId: list[ - Annotated[ - str, - Field( - pattern=r"^https://orcid\.org/[-X0-9]{9,21}$", - examples=["https://orcid.org/0000-0002-9079-593X"], - json_schema_extra={"format": "uri"}, - ), - ], - ] = [] + orcidId: list[OrcidIdStr] = [] class BasePerson(_OptionalLists): @@ -173,3 +171,41 @@ class PersonRuleSetResponse(_BaseRuleSet): Literal["PersonRuleSetResponse"], Field(alias="$type", frozen=True) ] = "PersonRuleSetResponse" stableTargetId: MergedPersonIdentifier + + +class PersonMapping(_Stem, BaseMapping): + """Mapping for describing a person transformation.""" + + entityType: Annotated[ + Literal["PersonMapping"], Field(alias="$type", frozen=True) + ] = "PersonMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + affiliation: list[MappingField[list[MergedOrganizationIdentifier]]] = [] + email: list[MappingField[list[Email]]] = [] + familyName: list[MappingField[list[FamilyNameStr]]] = [] + fullName: list[MappingField[list[FullNameStr]]] = [] + givenName: list[MappingField[list[GivenNameStr]]] = [] + isniId: list[MappingField[list[IsniIdStr]]] = [] + memberOf: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = [] + orcidId: list[MappingField[list[OrcidIdStr]]] = [] + + +class PersonFilter(_Stem, BaseFilter): + """Class for defining filter rules for person items.""" + + entityType: Annotated[ + Literal["PersonFilter"], Field(alias="$type", frozen=True) + ] = "PersonFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + affiliation: list[FilterField] = [] + email: list[FilterField] = [] + familyName: list[FilterField] = [] + fullName: list[FilterField] = [] + givenName: list[FilterField] = [] + isniId: list[FilterField] = [] + memberOf: list[FilterField] = [] + orcidId: list[FilterField] = [] diff --git a/mex/common/models/primary_source.py b/mex/common/models/primary_source.py index 46fa0945..e7a5592d 100644 --- a/mex/common/models/primary_source.py +++ b/mex/common/models/primary_source.py @@ -5,6 +5,8 @@ from pydantic import AfterValidator, Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -25,6 +27,19 @@ Text, ) +VersionStr = Annotated[ + str, + Field( + examples=["v1", "2023-01-16", "Schema 9"], + ), +] +AnyContactIdentifier = Annotated[ + MergedOrganizationalUnitIdentifier + | MergedPersonIdentifier + | MergedContactPointIdentifier, + AfterValidator(Identifier), +] + class _Stem(BaseModel): stemType: ClassVar[Annotated[Literal["PrimarySource"], Field(frozen=True)]] = ( @@ -34,14 +49,7 @@ class _Stem(BaseModel): class _OptionalLists(_Stem): alternativeTitle: list[Text] = [] - contact: list[ - Annotated[ - MergedOrganizationalUnitIdentifier - | MergedPersonIdentifier - | MergedContactPointIdentifier, - AfterValidator(Identifier), - ] - ] = [] + contact: list[AnyContactIdentifier] = [] description: list[Text] = [] documentation: list[Link] = [] locatedAt: list[Link] = [] @@ -50,26 +58,11 @@ class _OptionalLists(_Stem): class _OptionalValues(_Stem): - version: ( - Annotated[ - str, - Field( - examples=["v1", "2023-01-16", "Schema 9"], - ), - ] - | None - ) = None + version: VersionStr | None = None class _VariadicValues(_Stem): - version: list[ - Annotated[ - str, - Field( - examples=["v1", "2023-01-16", "Schema 9"], - ), - ] - ] = [] + version: list[VersionStr] = [] class BasePrimarySource(_OptionalLists, _OptionalValues): @@ -167,3 +160,41 @@ class PrimarySourceRuleSetResponse(_BaseRuleSet): Literal["PrimarySourceRuleSetResponse"], Field(alias="$type", frozen=True) ] = "PrimarySourceRuleSetResponse" stableTargetId: MergedPrimarySourceIdentifier + + +class PrimarySourceMapping(_Stem, BaseMapping): + """Mapping for describing a primary source transformation.""" + + entityType: Annotated[ + Literal["PrimarySourceMapping"], Field(alias="$type", frozen=True) + ] = "PrimarySourceMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + version: list[MappingField[VersionStr | None]] = [] + alternativeTitle: list[MappingField[list[Text]]] = [] + contact: list[MappingField[list[AnyContactIdentifier]]] = [] + description: list[MappingField[list[Text]]] = [] + documentation: list[MappingField[list[Link]]] = [] + locatedAt: list[MappingField[list[Link]]] = [] + title: list[MappingField[list[Text]]] = [] + unitInCharge: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = [] + + +class PrimarySourceFilter(_Stem, BaseFilter): + """Class for defining filter rules for primary source items.""" + + entityType: Annotated[ + Literal["PrimarySourceFilter"], Field(alias="$type", frozen=True) + ] = "PrimarySourceFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + alternativeTitle: list[FilterField] = [] + contact: list[FilterField] = [] + description: list[FilterField] = [] + documentation: list[FilterField] = [] + locatedAt: list[FilterField] = [] + title: list[FilterField] = [] + unitInCharge: list[FilterField] = [] + version: list[FilterField] = [] diff --git a/mex/common/models/resource.py b/mex/common/models/resource.py index 86dabb0e..ca2d0fc0 100644 --- a/mex/common/models/resource.py +++ b/mex/common/models/resource.py @@ -5,6 +5,8 @@ from pydantic import AfterValidator, Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -45,6 +47,7 @@ YearMonthDayTime, ) +ConformsToStr = Annotated[str, Field(examples=["FHIR", "LOINC", "SNOMED", "ICD-10"])] DoiStr = Annotated[ str, Field( @@ -66,6 +69,33 @@ json_schema_extra={"format": "uri"}, ), ] +MeshIdStr = Annotated[ + str, + Field( + pattern=r"^http://id\.nlm\.nih\.gov/mesh/[A-Z0-9]{2,64}$", + examples=["http://id.nlm.nih.gov/mesh/D001604"], + json_schema_extra={"format": "uri"}, + ), +] +TemporalStr = Annotated[ + str, + Field( + examples=[ + "2022-01 bis 2022-03", + "Sommer 2023", + "nach 2013", + "1998-2008", + ] + ), +] +AnyContactIdentifier = Annotated[ + MergedOrganizationalUnitIdentifier + | MergedPersonIdentifier + | MergedContactPointIdentifier, + AfterValidator(Identifier), +] +MaxTypicalAgeInt = Annotated[int, Field(examples=["99", "21"])] +MinTypicalAgeInt = Annotated[int, Field(examples=["0", "18"])] class _Stem(BaseModel): @@ -76,9 +106,7 @@ class _OptionalLists(_Stem): accessPlatform: list[MergedAccessPlatformIdentifier] = [] alternativeTitle: list[Text] = [] anonymizationPseudonymization: list[AnonymizationPseudonymization] = [] - conformsTo: list[ - Annotated[str, Field(examples=["FHIR", "LOINC", "SNOMED", "ICD-10"])] - ] = [] + conformsTo: list[ConformsToStr] = [] contributingUnit: list[MergedOrganizationalUnitIdentifier] = [] contributor: list[MergedPersonIdentifier] = [] creator: list[MergedPersonIdentifier] = [] @@ -93,16 +121,7 @@ class _OptionalLists(_Stem): keyword: list[Text] = [] language: list[Language] = [] loincId: list[LoincIdStr] = [] - meshId: list[ - Annotated[ - str, - Field( - pattern=r"^http://id\.nlm\.nih\.gov/mesh/[A-Z0-9]{2,64}$", - examples=["http://id.nlm.nih.gov/mesh/D001604"], - json_schema_extra={"format": "uri"}, - ), - ] - ] = [] + meshId: list[MeshIdStr] = [] method: list[Text] = [] methodDescription: list[Text] = [] populationCoverage: list[Text] = [] @@ -118,17 +137,7 @@ class _OptionalLists(_Stem): class _RequiredLists(_Stem): - contact: Annotated[ - list[ - Annotated[ - MergedOrganizationalUnitIdentifier - | MergedPersonIdentifier - | MergedContactPointIdentifier, - AfterValidator(Identifier), - ] - ], - Field(min_length=1), - ] + contact: Annotated[list[AnyContactIdentifier], Field(min_length=1)] theme: Annotated[list[Theme], Field(min_length=1)] title: Annotated[list[Text], Field(min_length=1)] unitInCharge: Annotated[ @@ -137,14 +146,7 @@ class _RequiredLists(_Stem): class _SparseLists(_Stem): - contact: list[ - Annotated[ - MergedOrganizationalUnitIdentifier - | MergedPersonIdentifier - | MergedContactPointIdentifier, - AfterValidator(Identifier), - ] - ] = [] + contact: list[AnyContactIdentifier] = [] theme: list[Theme] = [] title: list[Text] = [] unitInCharge: list[MergedOrganizationalUnitIdentifier] = [] @@ -156,27 +158,12 @@ class _OptionalValues(_Stem): doi: DoiStr | None = None hasPersonalData: PersonalData | None = None license: License | None = None - maxTypicalAge: Annotated[int, Field(examples=["99", "21"])] | None = None - minTypicalAge: Annotated[int, Field(examples=["0", "18"])] | None = None + maxTypicalAge: MaxTypicalAgeInt | None = None + minTypicalAge: MinTypicalAgeInt | None = None modified: YearMonthDayTime | YearMonthDay | YearMonth | Year | None = None sizeOfDataBasis: str | None = None temporal: ( - YearMonthDayTime - | YearMonthDay - | YearMonth - | Year - | Annotated[ - str, - Field( - examples=[ - "2022-01 bis 2022-03", - "Sommer 2023", - "nach 2013", - "1998-2008", - ] - ), - ] - | None + YearMonthDayTime | YearMonthDay | YearMonth | Year | TemporalStr | None ) = None wasGeneratedBy: MergedActivityIdentifier | None = None @@ -196,26 +183,12 @@ class _VariadicValues(_Stem): doi: list[DoiStr] = [] hasPersonalData: list[PersonalData] = [] license: list[License] = [] - maxTypicalAge: list[Annotated[int, Field(examples=["99", "21"])]] = [] - minTypicalAge: list[Annotated[int, Field(examples=["0", "18"])]] = [] + maxTypicalAge: list[MaxTypicalAgeInt] = [] + minTypicalAge: list[MinTypicalAgeInt] = [] modified: list[YearMonthDayTime | YearMonthDay | YearMonth | Year] = [] sizeOfDataBasis: list[str] = [] temporal: list[ - YearMonthDayTime - | YearMonthDay - | YearMonth - | Year - | Annotated[ - str, - Field( - examples=[ - "2022-01 bis 2022-03", - "Sommer 2023", - "nach 2013", - "1998-2008", - ] - ), - ] + YearMonthDayTime | YearMonthDay | YearMonth | Year | TemporalStr ] = [] wasGeneratedBy: list[MergedActivityIdentifier] = [] @@ -360,3 +333,137 @@ class ResourceRuleSetResponse(_BaseRuleSet): Literal["ResourceRuleSetResponse"], Field(alias="$type", frozen=True) ] = "ResourceRuleSetResponse" stableTargetId: MergedResourceIdentifier + + +class ResourceMapping(_Stem, BaseMapping): + """Mapping for describing a resource transformation.""" + + entityType: Annotated[ + Literal["ResourceMapping"], Field(alias="$type", frozen=True) + ] = "ResourceMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + accessRestriction: Annotated[ + list[MappingField[AccessRestriction]], Field(min_length=1) + ] + accrualPeriodicity: list[MappingField[Frequency | None]] = [] + created: list[ + MappingField[YearMonthDayTime | YearMonthDay | YearMonth | Year | None] + ] = [] + doi: list[MappingField[DoiStr | None]] = [] + hasPersonalData: list[MappingField[PersonalData | None]] = [] + license: list[MappingField[License | None]] = [] + maxTypicalAge: list[MappingField[MaxTypicalAgeInt | None]] = [] + minTypicalAge: list[MappingField[MinTypicalAgeInt | None]] = [] + modified: list[ + MappingField[YearMonthDayTime | YearMonthDay | YearMonth | Year | None] + ] = [] + sizeOfDataBasis: list[MappingField[str | None]] = [] + temporal: list[ + MappingField[ + YearMonthDayTime | YearMonthDay | YearMonth | Year | TemporalStr | None + ] + ] = [] + wasGeneratedBy: list[MappingField[MergedActivityIdentifier | None]] = [] + contact: Annotated[ + list[MappingField[list[AnyContactIdentifier]]], + Field(min_length=1), + ] + theme: Annotated[list[MappingField[list[Theme]]], Field(min_length=1)] + title: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + unitInCharge: Annotated[ + list[MappingField[list[MergedOrganizationalUnitIdentifier]]], + Field(min_length=1), + ] + accessPlatform: list[MappingField[list[MergedAccessPlatformIdentifier]]] = [] + alternativeTitle: list[MappingField[list[Text]]] = [] + anonymizationPseudonymization: list[ + MappingField[list[AnonymizationPseudonymization]] + ] = [] + conformsTo: list[MappingField[list[ConformsToStr]]] = [] + contributingUnit: list[MappingField[list[MergedOrganizationalUnitIdentifier]]] = [] + contributor: list[MappingField[list[MergedPersonIdentifier]]] = [] + creator: list[MappingField[list[MergedPersonIdentifier]]] = [] + description: list[MappingField[list[Text]]] = [] + distribution: list[MappingField[list[MergedDistributionIdentifier]]] = [] + documentation: list[MappingField[list[Link]]] = [] + externalPartner: list[MappingField[list[MergedOrganizationIdentifier]]] = [] + hasLegalBasis: list[MappingField[list[Text]]] = [] + icd10code: list[MappingField[list[str]]] = [] + instrumentToolOrApparatus: list[MappingField[list[Text]]] = [] + isPartOf: list[MappingField[list[MergedResourceIdentifier]]] = [] + keyword: list[MappingField[list[Text]]] = [] + language: list[MappingField[list[Language]]] = [] + loincId: list[MappingField[list[LoincIdStr]]] = [] + meshId: list[MappingField[list[MeshIdStr]]] = [] + method: list[MappingField[list[Text]]] = [] + methodDescription: list[MappingField[list[Text]]] = [] + populationCoverage: list[MappingField[list[Text]]] = [] + publication: list[MappingField[list[MergedBibliographicResourceIdentifier]]] = [] + publisher: list[MappingField[list[MergedOrganizationIdentifier]]] = [] + qualityInformation: list[MappingField[list[Text]]] = [] + resourceCreationMethod: list[MappingField[list[ResourceCreationMethod]]] = [] + resourceTypeGeneral: list[MappingField[list[ResourceTypeGeneral]]] = [] + resourceTypeSpecific: list[MappingField[list[Text]]] = [] + rights: list[MappingField[list[Text]]] = [] + spatial: list[MappingField[list[Text]]] = [] + stateOfDataProcessing: list[MappingField[list[DataProcessingState]]] = [] + + +class ResourceFilter(_Stem, BaseFilter): + """Class for defining filter rules for resource items.""" + + entityType: Annotated[ + Literal["ResourceFilter"], Field(alias="$type", frozen=True) + ] = "ResourceFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + accessPlatform: list[FilterField] = [] + accessRestriction: list[FilterField] = [] + accrualPeriodicity: list[FilterField] = [] + alternativeTitle: list[FilterField] = [] + anonymizationPseudonymization: list[FilterField] = [] + conformsTo: list[FilterField] = [] + contact: list[FilterField] = [] + contributingUnit: list[FilterField] = [] + contributor: list[FilterField] = [] + created: list[FilterField] = [] + doi: list[FilterField] = [] + creator: list[FilterField] = [] + description: list[FilterField] = [] + distribution: list[FilterField] = [] + documentation: list[FilterField] = [] + externalPartner: list[FilterField] = [] + hasLegalBasis: list[FilterField] = [] + hasPersonalData: list[FilterField] = [] + icd10code: list[FilterField] = [] + instrumentToolOrApparatus: list[FilterField] = [] + isPartOf: list[FilterField] = [] + keyword: list[FilterField] = [] + language: list[FilterField] = [] + license: list[FilterField] = [] + loincId: list[FilterField] = [] + maxTypicalAge: list[FilterField] = [] + meshId: list[FilterField] = [] + method: list[FilterField] = [] + methodDescription: list[FilterField] = [] + minTypicalAge: list[FilterField] = [] + modified: list[FilterField] = [] + populationCoverage: list[FilterField] = [] + publication: list[FilterField] = [] + publisher: list[FilterField] = [] + qualityInformation: list[FilterField] = [] + resourceCreationMethod: list[FilterField] = [] + resourceTypeGeneral: list[FilterField] = [] + resourceTypeSpecific: list[FilterField] = [] + rights: list[FilterField] = [] + sizeOfDataBasis: list[FilterField] = [] + spatial: list[FilterField] = [] + stateOfDataProcessing: list[FilterField] = [] + temporal: list[FilterField] = [] + theme: list[FilterField] = [] + title: list[FilterField] = [] + unitInCharge: list[FilterField] = [] + wasGeneratedBy: list[FilterField] = [] diff --git a/mex/common/models/variable.py b/mex/common/models/variable.py index 9114d0e6..3470168b 100644 --- a/mex/common/models/variable.py +++ b/mex/common/models/variable.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -31,6 +33,22 @@ str, Field(examples=["integer", "string", "image", "int55", "number"]), ] +ValueSetStr = Annotated[ + str, + Field( + examples=[ + "Ja, stark eingeschränkt", + "Ja, etwas eingeschränkt", + "Nein, überhaupt nicht eingeschränkt", + ], + ), +] +LabelText = Annotated[ + Text, + Field( + examples=[{"language": "de", "value": "Mehrere Treppenabsätze steigen"}], + ), +] class _Stem(BaseModel): @@ -40,48 +58,19 @@ class _Stem(BaseModel): class _OptionalLists(_Stem): belongsTo: list[MergedVariableGroupIdentifier] = [] description: list[Text] = [] - valueSet: list[ - Annotated[ - str, - Field( - examples=[ - "Ja, stark eingeschränkt", - "Ja, etwas eingeschränkt", - "Nein, überhaupt nicht eingeschränkt", - ], - ), - ] - ] = [] + valueSet: list[ValueSetStr] = [] class _RequiredLists(_Stem): label: Annotated[ - list[ - Annotated[ - Text, - Field( - examples=[ - {"language": "de", "value": "Mehrere Treppenabsätze steigen"} - ], - ), - ] - ], + list[LabelText], Field(min_length=1), ] usedIn: Annotated[list[MergedResourceIdentifier], Field(min_length=1)] class _SparseLists(_Stem): - label: list[ - Annotated[ - Text, - Field( - examples=[ - {"language": "de", "value": "Mehrere Treppenabsätze steigen"} - ], - ), - ] - ] = [] + label: list[LabelText] = [] usedIn: list[MergedResourceIdentifier] = [] @@ -191,3 +180,41 @@ class VariableRuleSetResponse(_BaseRuleSet): Literal["VariableRuleSetResponse"], Field(alias="$type", frozen=True) ] = "VariableRuleSetResponse" stableTargetId: MergedVariableIdentifier + + +class VariableMapping(_Stem, BaseMapping): + """Mapping for describing a variable transformation.""" + + entityType: Annotated[ + Literal["VariableMapping"], Field(alias="$type", frozen=True) + ] = "VariableMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + codingSystem: list[MappingField[CodingSystemStr | None]] = [] + dataType: list[MappingField[DataTypeStr | None]] = [] + label: Annotated[list[MappingField[list[LabelText]]], Field(min_length=1)] + usedIn: Annotated[ + list[MappingField[list[MergedResourceIdentifier]]], Field(min_length=1) + ] + belongsTo: list[MappingField[list[MergedVariableGroupIdentifier]]] = [] + description: list[MappingField[list[Text]]] = [] + valueSet: list[MappingField[list[ValueSetStr]]] = [] + + +class VariableFilter(_Stem, BaseFilter): + """Class for defining filter rules for variable items.""" + + entityType: Annotated[ + Literal["VariableFilter"], Field(alias="$type", frozen=True) + ] = "VariableFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + belongsTo: list[FilterField] = [] + codingSystem: list[FilterField] = [] + dataType: list[FilterField] = [] + description: list[FilterField] = [] + label: list[FilterField] = [] + usedIn: list[FilterField] = [] + valueSet: list[FilterField] = [] diff --git a/mex/common/models/variable_group.py b/mex/common/models/variable_group.py index 051b2291..e42b57af 100644 --- a/mex/common/models/variable_group.py +++ b/mex/common/models/variable_group.py @@ -5,6 +5,8 @@ from pydantic import Field, computed_field from mex.common.models.base.extracted_data import ExtractedData +from mex.common.models.base.filter import BaseFilter, FilterField +from mex.common.models.base.mapping import BaseMapping, MappingField from mex.common.models.base.merged_item import MergedItem from mex.common.models.base.model import BaseModel from mex.common.models.base.preview_item import PreviewItem @@ -128,3 +130,31 @@ class VariableGroupRuleSetResponse(_BaseRuleSet): Literal["VariableGroupRuleSetResponse"], Field(alias="$type", frozen=True) ] = "VariableGroupRuleSetResponse" stableTargetId: MergedVariableGroupIdentifier + + +class VariableGroupMapping(_Stem, BaseMapping): + """Mapping for describing a variable group transformation.""" + + entityType: Annotated[ + Literal["VariableGroupMapping"], Field(alias="$type", frozen=True) + ] = "VariableGroupMapping" + hadPrimarySource: Annotated[ + list[MappingField[MergedPrimarySourceIdentifier]], Field(min_length=1) + ] + identifierInPrimarySource: Annotated[list[MappingField[str]], Field(min_length=1)] + containedBy: Annotated[ + list[MappingField[list[MergedResourceIdentifier]]], Field(min_length=1) + ] + label: Annotated[list[MappingField[list[Text]]], Field(min_length=1)] + + +class VariableGroupFilter(_Stem, BaseFilter): + """Class for defining filter rules for variable group items.""" + + entityType: Annotated[ + Literal["VariableGroupFilter"], Field(alias="$type", frozen=True) + ] = "VariableGroupFilter" + hadPrimarySource: list[FilterField] = [] + identifierInPrimarySource: list[FilterField] = [] + containedBy: list[FilterField] = [] + label: list[FilterField] = [] diff --git a/tests/models/test_filter.py b/tests/models/test_filter.py index f3f8d54f..17ba6a8e 100644 --- a/tests/models/test_filter.py +++ b/tests/models/test_filter.py @@ -1,39 +1,50 @@ -from typing import Annotated, ClassVar, Literal +from mex.common.models import ( + EXTRACTED_MODEL_CLASSES, + EXTRACTED_MODEL_CLASSES_BY_NAME, + FILTER_MODEL_CLASSES, + FilterField, + PersonFilter, +) -from pydantic import Field -from mex.common.models import ExtractedData -from mex.common.models.base.filter import generate_entity_filter_schema -from mex.common.types import MergedOrganizationalUnitIdentifier -from mex.common.types.email import Email +def test_all_filter_classes_are_defined() -> None: + stem_types = sorted(c.stemType for c in EXTRACTED_MODEL_CLASSES) + assert sorted(c.stemType for c in FILTER_MODEL_CLASSES) == stem_types -class ExtractedDummy(ExtractedData): - stemType: ClassVar[Annotated[Literal["Dummy"], Field(frozen=True)]] = "Dummy" - entityType: Annotated[ - Literal["ExtractedDummy"], Field(alias="$type", frozen=True) - ] = "ExtractedDummy" - dummy_identifier: MergedOrganizationalUnitIdentifier | None = None # not required - dummy_str: str - dummy_int: int | None = None # not required - dummy_email: Email - dummy_list: list[str] = [] # not required - dummy_min_length_list: Annotated[list[str], Field(min_length=1)] +def test_all_filter_fields_are_defined() -> None: + for filter_cls in FILTER_MODEL_CLASSES: + extracted_cls = EXTRACTED_MODEL_CLASSES_BY_NAME[ + f"Extracted{filter_cls.stemType}" + ] + assert set(filter_cls.model_fields) == set(extracted_cls.model_fields) + field_defs = { + field_name: (field_info.annotation, field_info.default) + for field_name, field_info in filter_cls.model_fields.items() + if field_name != "entityType" + } + assert all( + (annotation, default) == (list[FilterField], []) + for annotation, default in field_defs.values() + ) -def test_entity_filter_schema() -> None: - schema_model = generate_entity_filter_schema(ExtractedDummy) - - expected = { +def test_filter_model_schema() -> None: + assert PersonFilter.model_json_schema() == { "$defs": { - "EntityFilter": { + "FilterField": { "additionalProperties": False, - "description": "Entity filter model.", + "description": "Entity filter field model.", "properties": { - "comment": { + "fieldInPrimarySource": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, + "title": "fieldInPrimarySource", + }, + "locationInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Comment", + "title": "locationInPrimarySource", }, "examplesInPrimarySource": { "anyOf": [ @@ -41,29 +52,25 @@ def test_entity_filter_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Examplesinprimarysource", - }, - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", + "title": "examplesInPrimarySource", }, "mappingRules": { - "items": {"$ref": "#/$defs/EntityFilterRule"}, + "items": {"$ref": "#/$defs/FilterRule"}, "minItems": 1, - "title": "Mappingrules", + "title": "mappingRules", "type": "array", }, + "comment": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, + "title": "comment", + }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "EntityFilter", + "required": ["mappingRules"], + "title": "FilterField", "type": "object", }, - "EntityFilterRule": { + "FilterRule": { "additionalProperties": False, "description": "Entity filter rule model.", "properties": { @@ -73,29 +80,89 @@ def test_entity_filter_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Forvalues", + "title": "forValues", }, "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", + "title": "rule", }, }, - "title": "EntityFilterRule", + "title": "FilterRule", "type": "object", }, }, - "description": "Schema for entity filters for the entity type ExtractedDummy.", + "additionalProperties": False, + "description": "Class for defining filter rules for person items.", "properties": { - "ExtractedDummy": { - "default": None, - "items": {"$ref": "#/$defs/EntityFilter"}, - "title": "Extracteddummy", + "$type": { + "const": "PersonFilter", + "default": "PersonFilter", + "enum": ["PersonFilter"], + "title": "$Type", + "type": "string", + }, + "hadPrimarySource": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Hadprimarysource", + "type": "array", + }, + "identifierInPrimarySource": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Identifierinprimarysource", + "type": "array", + }, + "affiliation": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Affiliation", + "type": "array", + }, + "email": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Email", "type": "array", - } + }, + "familyName": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Familyname", + "type": "array", + }, + "fullName": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Fullname", + "type": "array", + }, + "givenName": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Givenname", + "type": "array", + }, + "isniId": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Isniid", + "type": "array", + }, + "memberOf": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Memberof", + "type": "array", + }, + "orcidId": { + "default": [], + "items": {"$ref": "#/$defs/FilterField"}, + "title": "Orcidid", + "type": "array", + }, }, - "title": "DummyEntityFilter", + "title": "PersonFilter", "type": "object", } - - assert schema_model.model_json_schema() == expected diff --git a/tests/models/test_mapping.py b/tests/models/test_mapping.py index d9970694..5d6c193e 100644 --- a/tests/models/test_mapping.py +++ b/tests/models/test_mapping.py @@ -1,127 +1,56 @@ -from typing import Annotated, ClassVar, Literal +from typing import get_origin -from pydantic import Field +from pydantic_core import PydanticUndefined -from mex.common.models import ExtractedData -from mex.common.models.base.mapping import generate_mapping_schema -from mex.common.types import Email, Identifier, MergedOrganizationalUnitIdentifier +from mex.common.models import ( + EXTRACTED_MODEL_CLASSES, + EXTRACTED_MODEL_CLASSES_BY_NAME, + MAPPING_MODEL_CLASSES, + MappingField, + VariableGroupMapping, +) +from mex.common.testing import Joker -class ExtractedDummyIdentifier(Identifier): - pass +def test_all_mapping_classes_are_defined() -> None: + stem_types = sorted(c.stemType for c in EXTRACTED_MODEL_CLASSES) + assert sorted(c.stemType for c in MAPPING_MODEL_CLASSES) == stem_types -class MergedDummyIdentifier(Identifier): - pass +def test_all_mapping_fields_are_defined() -> None: + for mapping_cls in MAPPING_MODEL_CLASSES: + extracted_cls = EXTRACTED_MODEL_CLASSES_BY_NAME[ + f"Extracted{mapping_cls.stemType}" + ] + assert set(mapping_cls.model_fields) == set(extracted_cls.model_fields) + field_defs = { + field_name: (field_info.annotation, field_info.default) + for field_name, field_info in mapping_cls.model_fields.items() + if field_name != "entityType" + } + assert all( + get_origin(annotation) is list + and annotation.__args__[0].__bases__[0] is MappingField + and default in (PydanticUndefined, []) + for annotation, default in field_defs.values() + ) -class ExtractedDummy(ExtractedData): - stemType: ClassVar[Annotated[Literal["Dummy"], Field(frozen=True)]] = "Dummy" - entityType: Annotated[ - Literal["ExtractedDummy"], Field(alias="$type", frozen=True) - ] = "ExtractedDummy" - identifier: Annotated[ExtractedDummyIdentifier, Field(frozen=True)] - stableTargetId: MergedDummyIdentifier - dummy_unit: MergedOrganizationalUnitIdentifier | None = None # not required - dummy_str: str - dummy_int: int | None = None # not required - dummy_email: Email - dummy_list: list[str] = [] # not required - dummy_min_length_list: Annotated[list[str], Field(min_length=1)] - - -def test_generate_mapping_schema() -> None: - schema_model = generate_mapping_schema(ExtractedDummy) - - expected = { +def test_mapping_model_schema() -> None: + assert VariableGroupMapping.model_json_schema() == { "$defs": { - "Dummy_emailFieldsInPrimarySource": { + "MappingField_MergedPrimarySourceIdentifier_": { "additionalProperties": False, - "description": "Mapping schema for Dummy_email fields in primary source.", "properties": { "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", - }, - "examplesInPrimarySource": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Examplesinprimarysource", - }, - "mappingRules": { - "items": {"$ref": "#/$defs/Dummy_emailMappingRule"}, - "minItems": 1, - "title": "Mappingrules", - "type": "array", - }, - "comment": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Comment", - }, - }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "Dummy_emailFieldsInPrimarySource", - "type": "object", - }, - "Dummy_emailMappingRule": { - "additionalProperties": False, - "description": "Mapping rule schema of field Dummy_email.", - "properties": { - "forValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Forvalues", - }, - "setValues": { - "anyOf": [ - { - "items": { - "examples": ["info@rki.de"], - "format": "email", - "pattern": "^[^@ \\t\\r\\n]+@[^@ \\t\\r\\n]+\\.[^@ \\t\\r\\n]+$", - "title": "Email", - "type": "string", - }, - "type": "array", - }, - {"type": "null"}, - ], - "default": None, - "title": "Setvalues", - }, - "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", - }, - }, - "title": "Dummy_emailMappingRule", - "type": "object", - }, - "Dummy_intFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Dummy_int fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", + "title": "fieldInPrimarySource", }, "locationInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Locationinprimarysource", + "title": "locationInPrimarySource", }, "examplesInPrimarySource": { "anyOf": [ @@ -129,136 +58,38 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Examplesinprimarysource", + "title": "examplesInPrimarySource", }, "mappingRules": { - "items": {"$ref": "#/$defs/Dummy_intMappingRule"}, + "items": { + "$ref": "#/$defs/MappingRule_MergedPrimarySourceIdentifier_" + }, "minItems": 1, - "title": "Mappingrules", + "title": "mappingRules", "type": "array", }, "comment": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Comment", - }, - }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "Dummy_intFieldsInPrimarySource", - "type": "object", - }, - "Dummy_intMappingRule": { - "additionalProperties": False, - "description": "Mapping rule schema of field Dummy_int.", - "properties": { - "forValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Forvalues", - }, - "setValues": { - "anyOf": [ - { - "items": { - "anyOf": [{"type": "integer"}, {"type": "null"}] - }, - "type": "array", - }, - {"type": "null"}, - ], - "default": None, - "title": "Setvalues", - }, - "rule": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Rule", + "title": "comment", }, }, - "title": "Dummy_intMappingRule", + "required": ["mappingRules"], + "title": "MappingField[MergedPrimarySourceIdentifier]", "type": "object", }, - "Dummy_listFieldsInPrimarySource": { + "MappingField_list_MergedResourceIdentifier__": { "additionalProperties": False, - "description": "Mapping schema for Dummy_list fields in primary source.", "properties": { "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", - }, - "examplesInPrimarySource": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Examplesinprimarysource", - }, - "mappingRules": { - "items": {"$ref": "#/$defs/Dummy_listMappingRule"}, - "minItems": 1, - "title": "Mappingrules", - "type": "array", - }, - "comment": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Comment", - }, - }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "Dummy_listFieldsInPrimarySource", - "type": "object", - }, - "Dummy_listMappingRule": { - "additionalProperties": False, - "description": "Mapping rule schema of field Dummy_list.", - "properties": { - "forValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Forvalues", - }, - "setValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Setvalues", - }, - "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", - }, - }, - "title": "Dummy_listMappingRule", - "type": "object", - }, - "Dummy_min_length_listFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Dummy_min_length_list fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", + "title": "fieldInPrimarySource", }, "locationInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Locationinprimarysource", + "title": "locationInPrimarySource", }, "examplesInPrimarySource": { "anyOf": [ @@ -266,65 +97,38 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Examplesinprimarysource", + "title": "examplesInPrimarySource", }, "mappingRules": { - "items": {"$ref": "#/$defs/Dummy_min_length_listMappingRule"}, + "items": { + "$ref": "#/$defs/MappingRule_list_MergedResourceIdentifier__" + }, "minItems": 1, - "title": "Mappingrules", + "title": "mappingRules", "type": "array", }, "comment": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Comment", + "title": "comment", }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "Dummy_min_length_listFieldsInPrimarySource", + "required": ["mappingRules"], + "title": "MappingField[list[MergedResourceIdentifier]]", "type": "object", }, - "Dummy_min_length_listMappingRule": { + "MappingField_list_Text__": { "additionalProperties": False, - "description": "Mapping rule schema of field Dummy_min_length_list.", "properties": { - "forValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Forvalues", - }, - "setValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Setvalues", - }, - "rule": { + "fieldInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", - }, - }, - "title": "Dummy_min_length_listMappingRule", - "type": "object", - }, - "Dummy_strFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Dummy_str fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", + "title": "fieldInPrimarySource", }, "locationInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Locationinprimarysource", + "title": "locationInPrimarySource", }, "examplesInPrimarySource": { "anyOf": [ @@ -332,65 +136,36 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Examplesinprimarysource", + "title": "examplesInPrimarySource", }, "mappingRules": { - "items": {"$ref": "#/$defs/Dummy_strMappingRule"}, + "items": {"$ref": "#/$defs/MappingRule_list_Text__"}, "minItems": 1, - "title": "Mappingrules", + "title": "mappingRules", "type": "array", }, "comment": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Comment", + "title": "comment", }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "Dummy_strFieldsInPrimarySource", + "required": ["mappingRules"], + "title": "MappingField[list[Text]]", "type": "object", }, - "Dummy_strMappingRule": { + "MappingField_str_": { "additionalProperties": False, - "description": "Mapping rule schema of field Dummy_str.", "properties": { - "forValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Forvalues", - }, - "setValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Setvalues", - }, - "rule": { + "fieldInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", - }, - }, - "title": "Dummy_strMappingRule", - "type": "object", - }, - "Dummy_unitFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Dummy_unit fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", + "title": "fieldInPrimarySource", }, "locationInPrimarySource": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Locationinprimarysource", + "title": "locationInPrimarySource", }, "examplesInPrimarySource": { "anyOf": [ @@ -398,27 +173,26 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Examplesinprimarysource", + "title": "examplesInPrimarySource", }, "mappingRules": { - "items": {"$ref": "#/$defs/Dummy_unitMappingRule"}, + "items": {"$ref": "#/$defs/MappingRule_str_"}, "minItems": 1, - "title": "Mappingrules", + "title": "mappingRules", "type": "array", }, "comment": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Comment", + "title": "comment", }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "Dummy_unitFieldsInPrimarySource", + "required": ["mappingRules"], + "title": "MappingField[str]", "type": "object", }, - "Dummy_unitMappingRule": { + "MappingRule_MergedPrimarySourceIdentifier_": { "additionalProperties": False, - "description": "Mapping rule schema of field Dummy_unit.", "properties": { "forValues": { "anyOf": [ @@ -426,77 +200,34 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Forvalues", + "title": "forValues", }, "setValues": { "anyOf": [ { "items": { - "anyOf": [ - { - "pattern": "^[a-zA-Z0-9]{14,22}$", - "title": "MergedOrganizationalUnitIdentifier", - "type": "string", - }, - {"type": "null"}, - ] + "pattern": "^[a-zA-Z0-9]{14,22}$", + "title": "MergedPrimarySourceIdentifier", + "type": "string", }, "type": "array", }, {"type": "null"}, ], "default": None, - "title": "Setvalues", + "title": "setValues", }, "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", - }, - }, - "title": "Dummy_unitMappingRule", - "type": "object", - }, - "HadprimarysourceFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Hadprimarysource fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", - }, - "examplesInPrimarySource": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Examplesinprimarysource", - }, - "mappingRules": { - "items": {"$ref": "#/$defs/HadprimarysourceMappingRule"}, - "minItems": 1, - "title": "Mappingrules", - "type": "array", - }, - "comment": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Comment", + "title": "rule", }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "HadprimarysourceFieldsInPrimarySource", + "title": "MappingRule[MergedPrimarySourceIdentifier]", "type": "object", }, - "HadprimarysourceMappingRule": { + "MappingRule_list_MergedResourceIdentifier__": { "additionalProperties": False, - "description": "Mapping rule schema of field Hadprimarysource.", "properties": { "forValues": { "anyOf": [ @@ -504,72 +235,37 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Forvalues", + "title": "forValues", }, "setValues": { "anyOf": [ { "items": { - "pattern": "^[a-zA-Z0-9]{14,22}$", - "title": "MergedPrimarySourceIdentifier", - "type": "string", + "items": { + "pattern": "^[a-zA-Z0-9]{14,22}$", + "title": "MergedResourceIdentifier", + "type": "string", + }, + "type": "array", }, "type": "array", }, {"type": "null"}, ], "default": None, - "title": "Setvalues", + "title": "setValues", }, "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", - }, - }, - "title": "HadprimarysourceMappingRule", - "type": "object", - }, - "IdentifierFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Identifier fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", - }, - "examplesInPrimarySource": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Examplesinprimarysource", - }, - "mappingRules": { - "items": {"$ref": "#/$defs/IdentifierMappingRule"}, - "minItems": 1, - "title": "Mappingrules", - "type": "array", - }, - "comment": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Comment", + "title": "rule", }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "IdentifierFieldsInPrimarySource", + "title": "MappingRule[list[MergedResourceIdentifier]]", "type": "object", }, - "IdentifierMappingRule": { + "MappingRule_list_Text__": { "additionalProperties": False, - "description": "Mapping rule schema of field Identifier.", "properties": { "forValues": { "anyOf": [ @@ -577,74 +273,33 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Forvalues", + "title": "forValues", }, "setValues": { "anyOf": [ { "items": { - "pattern": "^[a-zA-Z0-9]{14,22}$", - "title": "ExtractedDummyIdentifier", - "type": "string", + "items": {"$ref": "#/$defs/Text"}, + "type": "array", }, "type": "array", }, {"type": "null"}, ], "default": None, - "title": "Setvalues", + "title": "setValues", }, "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", + "title": "rule", }, }, - "title": "IdentifierMappingRule", + "title": "MappingRule[list[Text]]", "type": "object", }, - "IdentifierinprimarysourceFieldsInPrimarySource": { + "MappingRule_str_": { "additionalProperties": False, - "description": "Mapping schema for Identifierinprimarysource fields in primary source.", - "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", - }, - "examplesInPrimarySource": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Examplesinprimarysource", - }, - "mappingRules": { - "items": { - "$ref": "#/$defs/IdentifierinprimarysourceMappingRule" - }, - "minItems": 1, - "title": "Mappingrules", - "type": "array", - }, - "comment": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Comment", - }, - }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "IdentifierinprimarysourceFieldsInPrimarySource", - "type": "object", - }, - "IdentifierinprimarysourceMappingRule": { - "additionalProperties": False, - "description": "Mapping rule schema of field Identifierinprimarysource.", "properties": { "forValues": { "anyOf": [ @@ -652,7 +307,7 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Forvalues", + "title": "forValues", }, "setValues": { "anyOf": [ @@ -660,160 +315,82 @@ def test_generate_mapping_schema() -> None: {"type": "null"}, ], "default": None, - "title": "Setvalues", + "title": "setValues", }, "rule": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": None, - "title": "Rule", + "title": "rule", }, }, - "title": "IdentifierinprimarysourceMappingRule", + "title": "MappingRule[str]", "type": "object", }, - "StabletargetidFieldsInPrimarySource": { - "additionalProperties": False, - "description": "Mapping schema for Stabletargetid fields in primary source.", + "Text": { + "description": Joker(), "properties": { - "fieldInPrimarySource": { - "title": "Fieldinprimarysource", - "type": "string", - }, - "locationInPrimarySource": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Locationinprimarysource", - }, - "examplesInPrimarySource": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], + "value": {"minLength": 1, "title": "Value", "type": "string"}, + "language": { + "anyOf": [{"$ref": "#/$defs/TextLanguage"}, {"type": "null"}], "default": None, - "title": "Examplesinprimarysource", - }, - "mappingRules": { - "items": {"$ref": "#/$defs/StabletargetidMappingRule"}, - "minItems": 1, - "title": "Mappingrules", - "type": "array", - }, - "comment": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Comment", }, }, - "required": ["fieldInPrimarySource", "mappingRules"], - "title": "StabletargetidFieldsInPrimarySource", + "required": ["value"], + "title": "Text", "type": "object", }, - "StabletargetidMappingRule": { - "additionalProperties": False, - "description": "Mapping rule schema of field Stabletargetid.", - "properties": { - "forValues": { - "anyOf": [ - {"items": {"type": "string"}, "type": "array"}, - {"type": "null"}, - ], - "default": None, - "title": "Forvalues", - }, - "setValues": { - "anyOf": [ - { - "items": { - "pattern": "^[a-zA-Z0-9]{14,22}$", - "title": "MergedDummyIdentifier", - "type": "string", - }, - "type": "array", - }, - {"type": "null"}, - ], - "default": None, - "title": "Setvalues", - }, - "rule": { - "anyOf": [{"type": "string"}, {"type": "null"}], - "default": None, - "title": "Rule", - }, - }, - "title": "StabletargetidMappingRule", - "type": "object", + "TextLanguage": { + "description": "Possible language tags for `Text` values.", + "enum": ["de", "en"], + "title": "TextLanguage", + "type": "string", }, }, - "description": "Schema for mapping the properties of the entity type ExtractedDummy.", + "additionalProperties": False, + "description": "Mapping for describing a variable group transformation.", "properties": { + "$type": { + "const": "VariableGroupMapping", + "default": "VariableGroupMapping", + "enum": ["VariableGroupMapping"], + "title": "$Type", + "type": "string", + }, "hadPrimarySource": { - "items": {"$ref": "#/$defs/HadprimarysourceFieldsInPrimarySource"}, + "items": { + "$ref": "#/$defs/MappingField_MergedPrimarySourceIdentifier_" + }, + "minItems": 1, "title": "Hadprimarysource", "type": "array", }, "identifierInPrimarySource": { - "items": { - "$ref": "#/$defs/IdentifierinprimarysourceFieldsInPrimarySource" - }, + "items": {"$ref": "#/$defs/MappingField_str_"}, + "minItems": 1, "title": "Identifierinprimarysource", "type": "array", }, - "identifier": { - "items": {"$ref": "#/$defs/IdentifierFieldsInPrimarySource"}, - "title": "Identifier", - "type": "array", - }, - "stableTargetId": { - "items": {"$ref": "#/$defs/StabletargetidFieldsInPrimarySource"}, - "title": "Stabletargetid", - "type": "array", - }, - "dummy_unit": { - "default": None, - "items": {"$ref": "#/$defs/Dummy_unitFieldsInPrimarySource"}, - "title": "Dummy Unit", - "type": "array", - }, - "dummy_str": { - "items": {"$ref": "#/$defs/Dummy_strFieldsInPrimarySource"}, - "title": "Dummy Str", - "type": "array", - }, - "dummy_int": { - "default": None, - "items": {"$ref": "#/$defs/Dummy_intFieldsInPrimarySource"}, - "title": "Dummy Int", - "type": "array", - }, - "dummy_email": { - "items": {"$ref": "#/$defs/Dummy_emailFieldsInPrimarySource"}, - "title": "Dummy Email", - "type": "array", - }, - "dummy_list": { - "default": None, - "items": {"$ref": "#/$defs/Dummy_listFieldsInPrimarySource"}, - "title": "Dummy List", + "containedBy": { + "items": { + "$ref": "#/$defs/MappingField_list_MergedResourceIdentifier__" + }, + "minItems": 1, + "title": "Containedby", "type": "array", }, - "dummy_min_length_list": { - "items": {"$ref": "#/$defs/Dummy_min_length_listFieldsInPrimarySource"}, - "title": "Dummy Min Length List", + "label": { + "items": {"$ref": "#/$defs/MappingField_list_Text__"}, + "minItems": 1, + "title": "Label", "type": "array", }, }, "required": [ "hadPrimarySource", "identifierInPrimarySource", - "identifier", - "stableTargetId", - "dummy_str", - "dummy_email", - "dummy_min_length_list", + "containedBy", + "label", ], - "title": "DummyMapping", + "title": "VariableGroupMapping", "type": "object", } - - assert schema_model.model_json_schema() == expected diff --git a/tests/models/test_rules.py b/tests/models/test_rules.py index 7dc4ff0a..8c7be8d4 100644 --- a/tests/models/test_rules.py +++ b/tests/models/test_rules.py @@ -1,6 +1,7 @@ from mex.common.models import ( ADDITIVE_MODEL_CLASSES, BASE_MODEL_CLASSES_BY_NAME, + EXTRACTED_MODEL_CLASSES, PREVENTIVE_MODEL_CLASSES, RULE_SET_REQUEST_CLASSES, RULE_SET_RESPONSE_CLASSES, @@ -9,6 +10,19 @@ from mex.common.types import Identifier, MergedPrimarySourceIdentifier +def test_all_rules_are_defined() -> None: + stem_types = sorted(c.stemType for c in EXTRACTED_MODEL_CLASSES) + for lookup in ( + ADDITIVE_MODEL_CLASSES, + BASE_MODEL_CLASSES_BY_NAME, + PREVENTIVE_MODEL_CLASSES, + RULE_SET_REQUEST_CLASSES, + RULE_SET_RESPONSE_CLASSES, + SUBTRACTIVE_MODEL_CLASSES, + ): + assert sorted(c.stemType for c in lookup) == stem_types + + def test_additive_models_define_same_fields_as_base_model() -> None: for additive_rule in ADDITIVE_MODEL_CLASSES: base_model_name = "Base" + additive_rule.stemType