perf: keep fetched blueprints. Custom blueprint 'get_document'

src/common/entity/validators.py

@@ -1,8 +1,7 @@
 from collections.abc import Callable
 from typing import Any, Literal
 
-from common.exceptions import ApplicationException, ValidationException
-from common.utils.logging import logger
+from common.exceptions import ValidationException
 from domain_classes.blueprint import Blueprint
 from domain_classes.blueprint_attribute import BlueprintAttribute
 from enums import SIMOS, BuiltinDataTypes
@@ -27,17 +26,13 @@ def is_blueprint_instance_of(
                 the blueprint extends a blueprint that fulfills one of these three rules
             Otherwise it returns false.
     """
-    try:
-        if minimum_blueprint_type == BuiltinDataTypes.OBJECT.value:
+    if minimum_blueprint_type == BuiltinDataTypes.OBJECT.value:
+        return True
+    if minimum_blueprint_type == blueprint_type:
+        return True
+    for inherited_type in get_blueprint(blueprint_type).extends:
+        if is_blueprint_instance_of(minimum_blueprint_type, inherited_type, get_blueprint):
             return True
-        if minimum_blueprint_type == blueprint_type:
-            return True
-        for inherited_type in get_blueprint(blueprint_type).extends:
-            if is_blueprint_instance_of(minimum_blueprint_type, inherited_type, get_blueprint):
-                return True
-    except ApplicationException as ex:
-        logger.warn(ex)
-        return False
     return False
 
 

src/common/providers/blueprint_provider.py

@@ -4,14 +4,14 @@
 
 from authentication.models import User
 from common.address import Address
-from common.exceptions import NotFoundException
+from common.exceptions import ApplicationException, NotFoundException
 from common.providers.address_resolver.address_resolver import (
-    ResolvedAddress,
     resolve_address,
 )
 from common.utils.logging import logger
 from config import config
 from domain_classes.blueprint import Blueprint
+from enums import SIMOS
 from storage.data_source_interface import DataSource
 from storage.internal.data_source_repository import get_data_source
 
@@ -20,6 +20,36 @@ def substitute_get_blueprint(*args, **kwargs):
     raise ValueError("'get_blueprint' should not be called when fetching blueprints")
 
 
+def find_entity_by_name_in_package(
+    package: dict,
+    path_elements: list[str],
+    data_source: str,
+    get_data_source: Callable,
+    cache: dict,
+    concated_path: str,
+) -> dict:
+    for reference in package.get("content", []):
+        resolved_reference = resolve_address(Address(reference["address"], data_source), get_data_source).entity
+
+        # Add resolved reference to cache if they are blueprints
+        if resolved_reference["type"] in (SIMOS.BLUEPRINT.value, SIMOS.ENUM.value):
+            cache[f"{concated_path}/{resolved_reference['name']}"] = resolved_reference
+            if len(path_elements) == 1 and resolved_reference.get("name") == path_elements[0]:
+                return resolved_reference
+
+        if resolved_reference.get("name") == path_elements[0] and len(path_elements) > 1:
+            return find_entity_by_name_in_package(
+                resolved_reference,
+                path_elements[1:],
+                data_source,
+                get_data_source,
+                cache,
+                f"{concated_path}/{resolved_reference['name']}",
+            )
+
+    raise NotFoundException(f"Could not find entity with name '{path_elements[0]}' in package '{package['name']}'")
+
+
 class BlueprintProvider:
     def __init__(
         self,
@@ -31,9 +61,11 @@ def __init__(
         self.get_data_source = get_data_source
         self.resolve_address = resolve_address
         self.id = uuid.uuid4()
+        # Fetched blueprint documents are cached in this object, even before they are requested
+        self.prefetched_blueprints: dict[str, dict] = {}
 
     @lru_cache(maxsize=128)  # noqa B019
-    def get_data_source_cached(self, data_source_id: str, user: User) -> DataSource:
+    def get_data_source_cached(self, data_source_id: str) -> DataSource:
         # BlueprintProvider needs its own  'get_data_source' function to avoid circular imports
         return self.get_data_source(data_source_id, self.user, substitute_get_blueprint)
 
@@ -45,26 +77,41 @@ def get_blueprint_with_extended_attributes(self, type: str) -> Blueprint:
 
     @lru_cache(maxsize=config.CACHE_MAX_SIZE)  # noqa: B019
     def get_blueprint(self, type: str) -> Blueprint:
-        logger.debug(f"Cache miss! Fetching blueprint '{type}' '{hash(self)}'")
+        """Custom 'get_document' function that caches the fetched blueprints,
+        even if they were not the requested blueprint.
+        This is done for performance optimization, as often, all blueprints are requested at one point.
+        Only supports references on the "path" format.
+        """
+        if type in self.prefetched_blueprints:
+            logger.debug(f"Cache hit! Returning pre-fetched blueprint '{type}'")
+            return Blueprint(self.prefetched_blueprints[type], type)
         try:
-            resolved_address: ResolvedAddress = self.resolve_address(
-                Address.from_absolute(type),
-                lambda data_source_name: self.get_data_source_cached(data_source_name, self.user),
+            logger.debug(f"Cache miss! Fetching blueprint '{type}' '{hash(self)}'")
+            address = Address.from_absolute(type)
+            data_source = self.get_data_source_cached(address.data_source)
+            path_elements = address.path.split("/")
+            root_package = data_source.find({"name": path_elements[0], "type": SIMOS.PACKAGE.value, "isRoot": True})
+            if not root_package:
+                raise NotFoundException(f"Could not find root package '{path_elements[0]}'")
+            if len(root_package) > 1:
+                raise ApplicationException(f"Multiple root packages found with name '{path_elements[0]}'")
+            return Blueprint(
+                find_entity_by_name_in_package(
+                    root_package[0],
+                    path_elements[1:],
+                    address.data_source,
+                    get_data_source=lambda data_source_name: self.get_data_source_cached(data_source_name),
+                    cache=self.prefetched_blueprints,
+                    concated_path=f"dmss://{address.data_source}/{path_elements[0]}",
+                ),
+                type,
             )
+
         except NotFoundException as ex:
             raise NotFoundException(
                 f"Blueprint referenced with '{type}' could not be found. Make sure the reference is correct.",
                 data=ex.dict(),
             ) from ex
-        resolved_address = self.resolve_address(
-            Address.from_relative(
-                resolved_address.entity["address"],
-                resolved_address.document_id,
-                resolved_address.data_source_id,
-            ),
-            lambda data_source_name: self.get_data_source_cached(data_source_name, self.user),
-        )
-        return Blueprint(resolved_address.entity, type)
 
     def invalidate_cache(self):
         try:

src/enums.py

@@ -41,6 +41,7 @@ class StorageDataTypes(str, Enum):
 
 
 class SIMOS(Enum):
+    ENUM = "dmss://system/SIMOS/Enum"
     BLUEPRINT = "dmss://system/SIMOS/Blueprint"
     STORAGE_RECIPE = "dmss://system/SIMOS/StorageRecipe"
     STORAGE_ATTRIBUTE = "dmss://system/SIMOS/StorageAttribute"

src/storage/repositories/mongo.py

@@ -2,7 +2,7 @@
 
 import gridfs
 from pymongo import MongoClient
-from pymongo.errors import DuplicateKeyError, WriteError
+from pymongo.errors import DuplicateKeyError, OperationFailure, WriteError
 
 from common.exceptions import BadRequestException, NotFoundException
 from common.utils.encryption import decrypt
@@ -65,7 +65,7 @@ def update_blob(self, uid: str, blob: bytearray):
                 attempts += 1
                 response = self.blob_handler.put(blob, _id=uid)
                 return response
-            except WriteError as error:  # Likely caused by MongoDB rate limiting.
+            except (WriteError, OperationFailure) as error:  # Likely caused by MongoDB rate limiting.
                 logger.warning(f"Failed to upload blob (attempt: {attempts}), will retry:\n\t{error}")
                 sleep(2)
                 if attempts > 2:

src/tests/bdd/document/add_attribute_to_document.feature

@@ -188,7 +188,7 @@ Feature: Add attribute to document
       """
       {
         "name": "SignalGeneratorJob",
-        "type": "CORE:Blueprint",
+        "type": "dmss://system/SIMOS/Blueprint",
         "description": "",
         "extends": [
           "dmss://data-source-name/root_package/JobHandler"