diff --git a/src/common/entity/validators.py b/src/common/entity/validators.py index 4689e178..e1b44853 100644 --- a/src/common/entity/validators.py +++ b/src/common/entity/validators.py @@ -1,8 +1,7 @@ from collections.abc import Callable from typing import Any, Literal -from common.exceptions import ApplicationException, ValidationException -from common.utils.logging import logger +from common.exceptions import ValidationException from domain_classes.blueprint import Blueprint from domain_classes.blueprint_attribute import BlueprintAttribute from enums import SIMOS, BuiltinDataTypes @@ -27,17 +26,13 @@ def is_blueprint_instance_of( the blueprint extends a blueprint that fulfills one of these three rules Otherwise it returns false. """ - try: - if minimum_blueprint_type == BuiltinDataTypes.OBJECT.value: + if minimum_blueprint_type == BuiltinDataTypes.OBJECT.value: + return True + if minimum_blueprint_type == blueprint_type: + return True + for inherited_type in get_blueprint(blueprint_type).extends: + if is_blueprint_instance_of(minimum_blueprint_type, inherited_type, get_blueprint): return True - if minimum_blueprint_type == blueprint_type: - return True - for inherited_type in get_blueprint(blueprint_type).extends: - if is_blueprint_instance_of(minimum_blueprint_type, inherited_type, get_blueprint): - return True - except ApplicationException as ex: - logger.warn(ex) - return False return False diff --git a/src/common/providers/blueprint_provider.py b/src/common/providers/blueprint_provider.py index eedb1efa..24d8576a 100644 --- a/src/common/providers/blueprint_provider.py +++ b/src/common/providers/blueprint_provider.py @@ -4,14 +4,14 @@ from authentication.models import User from common.address import Address -from common.exceptions import NotFoundException +from common.exceptions import ApplicationException, NotFoundException from common.providers.address_resolver.address_resolver import ( - ResolvedAddress, resolve_address, ) from common.utils.logging import logger from config import config from domain_classes.blueprint import Blueprint +from enums import SIMOS from storage.data_source_interface import DataSource from storage.internal.data_source_repository import get_data_source @@ -20,6 +20,36 @@ def substitute_get_blueprint(*args, **kwargs): raise ValueError("'get_blueprint' should not be called when fetching blueprints") +def find_entity_by_name_in_package( + package: dict, + path_elements: list[str], + data_source: str, + get_data_source: Callable, + cache: dict, + concated_path: str, +) -> dict: + for reference in package.get("content", []): + resolved_reference = resolve_address(Address(reference["address"], data_source), get_data_source).entity + + # Add resolved reference to cache if they are blueprints + if resolved_reference["type"] in (SIMOS.BLUEPRINT.value, SIMOS.ENUM.value): + cache[f"{concated_path}/{resolved_reference['name']}"] = resolved_reference + if len(path_elements) == 1 and resolved_reference.get("name") == path_elements[0]: + return resolved_reference + + if resolved_reference.get("name") == path_elements[0] and len(path_elements) > 1: + return find_entity_by_name_in_package( + resolved_reference, + path_elements[1:], + data_source, + get_data_source, + cache, + f"{concated_path}/{resolved_reference['name']}", + ) + + raise NotFoundException(f"Could not find entity with name '{path_elements[0]}' in package '{package['name']}'") + + class BlueprintProvider: def __init__( self, @@ -31,9 +61,11 @@ def __init__( self.get_data_source = get_data_source self.resolve_address = resolve_address self.id = uuid.uuid4() + # Fetched blueprint documents are cached in this object, even before they are requested + self.prefetched_blueprints: dict[str, dict] = {} @lru_cache(maxsize=128) # noqa B019 - def get_data_source_cached(self, data_source_id: str, user: User) -> DataSource: + def get_data_source_cached(self, data_source_id: str) -> DataSource: # BlueprintProvider needs its own 'get_data_source' function to avoid circular imports return self.get_data_source(data_source_id, self.user, substitute_get_blueprint) @@ -45,26 +77,41 @@ def get_blueprint_with_extended_attributes(self, type: str) -> Blueprint: @lru_cache(maxsize=config.CACHE_MAX_SIZE) # noqa: B019 def get_blueprint(self, type: str) -> Blueprint: - logger.debug(f"Cache miss! Fetching blueprint '{type}' '{hash(self)}'") + """Custom 'get_document' function that caches the fetched blueprints, + even if they were not the requested blueprint. + This is done for performance optimization, as often, all blueprints are requested at one point. + Only supports references on the "path" format. + """ + if type in self.prefetched_blueprints: + logger.debug(f"Cache hit! Returning pre-fetched blueprint '{type}'") + return Blueprint(self.prefetched_blueprints[type], type) try: - resolved_address: ResolvedAddress = self.resolve_address( - Address.from_absolute(type), - lambda data_source_name: self.get_data_source_cached(data_source_name, self.user), + logger.debug(f"Cache miss! Fetching blueprint '{type}' '{hash(self)}'") + address = Address.from_absolute(type) + data_source = self.get_data_source_cached(address.data_source) + path_elements = address.path.split("/") + root_package = data_source.find({"name": path_elements[0], "type": SIMOS.PACKAGE.value, "isRoot": True}) + if not root_package: + raise NotFoundException(f"Could not find root package '{path_elements[0]}'") + if len(root_package) > 1: + raise ApplicationException(f"Multiple root packages found with name '{path_elements[0]}'") + return Blueprint( + find_entity_by_name_in_package( + root_package[0], + path_elements[1:], + address.data_source, + get_data_source=lambda data_source_name: self.get_data_source_cached(data_source_name), + cache=self.prefetched_blueprints, + concated_path=f"dmss://{address.data_source}/{path_elements[0]}", + ), + type, ) + except NotFoundException as ex: raise NotFoundException( f"Blueprint referenced with '{type}' could not be found. Make sure the reference is correct.", data=ex.dict(), ) from ex - resolved_address = self.resolve_address( - Address.from_relative( - resolved_address.entity["address"], - resolved_address.document_id, - resolved_address.data_source_id, - ), - lambda data_source_name: self.get_data_source_cached(data_source_name, self.user), - ) - return Blueprint(resolved_address.entity, type) def invalidate_cache(self): try: diff --git a/src/enums.py b/src/enums.py index b6860e7b..694d5333 100644 --- a/src/enums.py +++ b/src/enums.py @@ -41,6 +41,7 @@ class StorageDataTypes(str, Enum): class SIMOS(Enum): + ENUM = "dmss://system/SIMOS/Enum" BLUEPRINT = "dmss://system/SIMOS/Blueprint" STORAGE_RECIPE = "dmss://system/SIMOS/StorageRecipe" STORAGE_ATTRIBUTE = "dmss://system/SIMOS/StorageAttribute" diff --git a/src/storage/repositories/mongo.py b/src/storage/repositories/mongo.py index 839babb8..7e3e8d9c 100644 --- a/src/storage/repositories/mongo.py +++ b/src/storage/repositories/mongo.py @@ -2,7 +2,7 @@ import gridfs from pymongo import MongoClient -from pymongo.errors import DuplicateKeyError, WriteError +from pymongo.errors import DuplicateKeyError, OperationFailure, WriteError from common.exceptions import BadRequestException, NotFoundException from common.utils.encryption import decrypt @@ -65,7 +65,7 @@ def update_blob(self, uid: str, blob: bytearray): attempts += 1 response = self.blob_handler.put(blob, _id=uid) return response - except WriteError as error: # Likely caused by MongoDB rate limiting. + except (WriteError, OperationFailure) as error: # Likely caused by MongoDB rate limiting. logger.warning(f"Failed to upload blob (attempt: {attempts}), will retry:\n\t{error}") sleep(2) if attempts > 2: diff --git a/src/tests/bdd/document/add_attribute_to_document.feature b/src/tests/bdd/document/add_attribute_to_document.feature index f8c06ed5..5c2e00af 100644 --- a/src/tests/bdd/document/add_attribute_to_document.feature +++ b/src/tests/bdd/document/add_attribute_to_document.feature @@ -188,7 +188,7 @@ Feature: Add attribute to document """ { "name": "SignalGeneratorJob", - "type": "CORE:Blueprint", + "type": "dmss://system/SIMOS/Blueprint", "description": "", "extends": [ "dmss://data-source-name/root_package/JobHandler"