add module loader

vedro/_config.py

@@ -21,6 +21,8 @@
 from vedro.core import (
     Dispatcher,
     Factory,
+    ModuleFileLoader,
+    ModuleLoader,
     MonotonicScenarioRunner,
     MonotonicScenarioScheduler,
     MultiScenarioDiscoverer,
@@ -53,12 +55,16 @@ class Config(core.Config):
     class Registry(core.Config.Registry):
         Dispatcher = Singleton[Dispatcher](Dispatcher)
 
+        ModuleLoader = Factory[ModuleLoader](ModuleFileLoader)
+
         ScenarioFinder = Factory[ScenarioFinder](lambda: ScenarioFileFinder(
             file_filter=AnyFilter([HiddenFilter(), DunderFilter(), ExtFilter(only=["py"])]),
             dir_filter=AnyFilter([HiddenFilter(), DunderFilter()])
         ))
 
-        ScenarioLoader = Factory[ScenarioLoader](ScenarioFileLoader)
+        ScenarioLoader = Factory[ScenarioLoader](lambda: ScenarioFileLoader(
+            module_loader=Config.Registry.ModuleLoader(),
+        ))
 
         ScenarioOrderer = Factory[ScenarioOrderer](StableScenarioOrderer)
 

vedro/core/module_loader/_module_file_loader.py

@@ -2,6 +2,7 @@
 import importlib.util
 from importlib.abc import Loader
 from importlib.machinery import ModuleSpec
+from keyword import iskeyword
 from pathlib import Path
 from types import ModuleType
 from typing import cast
@@ -13,18 +14,31 @@
 
 class ModuleFileLoader(ModuleLoader):
     """
-    A loader class for loading Python modules from file paths.
+    Loads a module from a file path with optional validation of module names.
 
-    This class extends ModuleLoader to provide functionality for loading modules
-    from filesystem paths.
+    This class implements the ModuleLoader abstract base class, providing functionality
+    to load a module from a specified file path. Optionally, it can validate the module
+    names derived from the file paths to ensure they are valid Python identifiers.
     """
 
+    def __init__(self, *, validate_module_names: bool = False) -> None:
+        """
+        Initialize the ModuleFileLoader with optional module name validation.
+
+        :param validate_module_names: Flag to indicate whether module names should be
+                                      validated. Defaults to False.
+        """
+        self._validate_module_names = validate_module_names
+
     async def load(self, path: Path) -> ModuleType:
         """
-        Load a module from a file path.
+        Load a module from the specified file path.
 
-        :param path: The file path of the module to be loaded.
+        :param path: The file path to load the module from.
         :return: The loaded module.
+        :raises ValueError: If the module name derived from the path is invalid and
+                            validation is enabled.
+        :raises ModuleNotFoundError: If the module spec could not be created from the path.
         """
         spec = self._spec_from_path(path)
         module = self._module_from_spec(spec)
@@ -33,20 +47,41 @@ async def load(self, path: Path) -> ModuleType:
 
     def _path_to_module_name(self, path: Path) -> str:
         """
-        Convert a file path to a module name.
+        Convert a file path to a module name, optionally validating the name.
 
-        :param path: The file path to be converted.
-        :return: The corresponding module name.
+        :param path: The file path to convert.
+        :return: The derived module name.
+        :raises ValueError: If the module name derived from the path is invalid and
+                            validation is enabled.
         """
-        return ".".join(path.with_suffix("").parts)
+        parts = path.with_suffix("").parts
+        if self._validate_module_names:
+            for part in parts:
+                if not self._is_valid_identifier(part):
+                    raise ValueError(
+                        f"The module name derived from the path '{path}' is invalid "
+                        f"due to the segment '{part}'. A valid module name should "
+                        "start with a letter or underscore, contain only letters, "
+                        "digits, or underscores, and not be a Python keyword."
+                    )
+        return ".".join(parts)
+
+    def _is_valid_identifier(self, name: str) -> bool:
+        """
+        Check if a string is a valid Python identifier and not a keyword.
+
+        :param name: The string to check.
+        :return: True if the string is a valid identifier, False otherwise.
+        """
+        return name.isidentifier() and not iskeyword(name)
 
     def _spec_from_path(self, path: Path) -> ModuleSpec:
         """
         Create a module specification from a file path.
 
-        :param path: The file path for which to create the module specification.
-        :return: The module specification for the given path.
-        :raises ModuleNotFoundError: If no module specification can be created from the path.
+        :param path: The file path to create the spec from.
+        :return: The created module specification.
+        :raises ModuleNotFoundError: If the module spec could not be created from the path.
         """
         module_name = self._path_to_module_name(path)
         spec = importlib.util.spec_from_file_location(module_name, path)
@@ -56,18 +91,18 @@ def _spec_from_path(self, path: Path) -> ModuleSpec:
 
     def _module_from_spec(self, spec: ModuleSpec) -> ModuleType:
         """
-        Load a module from a module specification.
+        Create a module from a module specification.
 
-        :param spec: The module specification from which to load the module.
-        :return: The loaded module.
+        :param spec: The module specification to create the module from.
+        :return: The created module.
         """
         return importlib.util.module_from_spec(spec)
 
     def _exec_module(self, loader: Loader, module: ModuleType) -> None:
         """
-        Execute a loaded module.
+        Execute a module using its loader.
 
         :param loader: The loader to use for executing the module.
-        :param module: The module to be executed.
+        :param module: The module to execute.
         """
         loader.exec_module(module)

vedro/core/module_loader/_module_loader.py

@@ -7,16 +7,20 @@
 
 class ModuleLoader(ABC):
     """
-    Abstract base class for a module loader.
+    Represents an abstract base class for module loading.
 
-    This class defines an interface for loading Python modules.
+    This class defines the interface for loading a module from a given file path.
+    Subclasses must implement the `load` method to provide the specific module loading
+    functionality.
     """
 
     @abstractmethod
     async def load(self, path: Path) -> ModuleType:
         """
-        Load a module from a given path.
+        Load a module from the specified file path.
 
-        :param path: The file path of the module to be loaded.
+        :param path: The file path to load the module from.
+        :return: The loaded module.
+        :raises NotImplementedError: If the method is not implemented by a subclass.
         """
         pass

vedro/core/scenario_discoverer/_create_vscenario.py

@@ -28,3 +28,6 @@ def create_vscenario(scenario: Type[Scenario]) -> VirtualScenario:
             continue
         steps.append(VirtualStep(method))
     return VirtualScenario(scenario, steps)
+
+# TODO: Move the create_vscenario method to the ScenarioLoader class in v2.
+#       This will allow the ScenarioLoader to be responsible for creating VirtualScenario objects.

vedro/core/scenario_loader/_scenario_file_loader.py

@@ -1,44 +1,21 @@
-import importlib
-import importlib.util
 import os
-from importlib.abc import Loader
-from importlib.machinery import ModuleSpec
 from inspect import isclass
-from keyword import iskeyword
 from pathlib import Path
-from types import ModuleType
-from typing import Any, List, Type, cast
+from typing import Any, List, Optional, Type
 
 from ..._scenario import Scenario
+from ..module_loader import ModuleFileLoader, ModuleLoader
 from ._scenario_loader import ScenarioLoader
 
 __all__ = ("ScenarioFileLoader",)
 
 
 class ScenarioFileLoader(ScenarioLoader):
-    """
-    A class responsible for loading Vedro scenarios from a file.
-    """
-
-    def __init__(self) -> None:
-        """
-        Initialize the ScenarioFileLoader.
-
-        The loader is responsible for loading Vedro scenarios from a file.
-        """
-        self._validate_module_names: bool = False
+    def __init__(self, module_loader: Optional[ModuleLoader] = None) -> None:
+        self._module_loader = module_loader or ModuleFileLoader()  # backward compatibility
 
     async def load(self, path: Path) -> List[Type[Scenario]]:
-        """
-        Load Vedro scenarios from a module at the given path.
-
-        :param path: The file path of the module to load scenarios from.
-        :return: A list of loaded Vedro scenario classes.
-        :raises ValueError: If no valid Vedro scenarios are found in the module.
-        """
-        spec = self._spec_from_path(path)
-        module = self._module_from_spec(spec)
-        self._exec_module(cast(Loader, spec.loader), module)
+        module = await self._module_loader.load(path)
 
         loaded = []
         # Iterate over the module's dictionary because it preserves the order of definitions,
@@ -58,78 +35,7 @@ async def load(self, path: Path) -> List[Type[Scenario]]:
             )
         return loaded
 
-    def _path_to_module_name(self, path: Path) -> str:
-        """
-        Convert a file path to a valid Python module name.
-
-        :param path: The file path to convert.
-        :return: A string representing the module name.
-        :raises ValueError: If any part of the path is not a valid Python identifier.
-        """
-        parts = path.with_suffix("").parts
-        if self._validate_module_names:
-            for part in parts:
-                if not self._is_valid_identifier(part):
-                    raise ValueError(
-                        f"The module name derived from the path '{path}' is invalid "
-                        f"due to the segment '{part}'. A valid module name should "
-                        "start with a letter or underscore, contain only letters, "
-                        "digits, or underscores, and not be a Python keyword."
-                    )
-        return ".".join(parts)
-
-    def _is_valid_identifier(self, name: str) -> bool:
-        """
-        Check if a string is a valid Python identifier.
-
-        :param name: The string to check.
-        :return: True if the string is a valid identifier, False otherwise.
-        """
-        return name.isidentifier() and not iskeyword(name)
-
-    def _spec_from_path(self, path: Path) -> ModuleSpec:
-        """
-        Create a module specification from a file path.
-
-        :param path: The file path for which to create the module spec.
-        :return: The ModuleSpec for the given path.
-        :raises ModuleNotFoundError: If no module specification can be created for the path.
-        """
-        module_name = self._path_to_module_name(path)
-        spec = importlib.util.spec_from_file_location(module_name, path)
-        if spec is None:
-            raise ModuleNotFoundError(module_name)
-        return spec
-
-    def _module_from_spec(self, spec: ModuleSpec) -> ModuleType:
-        """
-        Load a module from a module specification.
-
-        :param spec: The module specification from which to load the module.
-        :return: The loaded module.
-        """
-        return importlib.util.module_from_spec(spec)
-
-    def _exec_module(self, loader: Loader, module: ModuleType) -> None:
-        """
-        Execute a module that has been loaded.
-
-        :param loader: The loader to use for executing the module.
-        :param module: The module to execute.
-        """
-        loader.exec_module(module)
-
     def _is_vedro_scenario(self, val: Any) -> bool:
-        """
-        Determine if a given value is a Vedro scenario class.
-
-        :param val: The value to check.
-        :return: True if the value is a Vedro scenario class, False otherwise.
-        :raises TypeError: If the value has a name suggesting it's a scenario but
-            doesn't inherit from 'vedro.Scenario'.
-        :raises ValueError: If the value inherits from 'vedro.Scenario' but its
-            name doesn't follow the naming convention.
-        """
         # First, check if 'val' is a class. Non-class values are not scenarios
         if not isclass(val):
             return False
@@ -141,27 +47,13 @@ def _is_vedro_scenario(self, val: Any) -> bool:
         if (val == Scenario) or (cls_name == "VedroTemplate"):
             return False
 
-        # Check if 'val' is a subclass of 'Scenario' for inheritance validation
-        is_scenario_subclass = issubclass(val, Scenario)
-
-        # Check if the name of 'val' follows the naming convention for scenarios
-        # It should start or end with 'Scenario'
-        has_scenario_in_name = cls_name.startswith("Scenario") or cls_name.endswith("Scenario")
-
-        # If 'val' follows both naming convention and is a subclass of 'Scenario',
-        # it's a valid scenario
-        if has_scenario_in_name and is_scenario_subclass:
+        # Check if 'val' is a subclass of Vedro's Scenario class
+        if issubclass(val, Scenario):
             return True
 
-        # If only the naming convention is followed, raise an error indicating inheritance issue
-        elif has_scenario_in_name:
+        # Raise an error if a class name suggests it's a scenario, but
+        # it doesn't inherit from Vedro.Scenario
+        if cls_name.startswith("Scenario") or cls_name.endswith("Scenario"):
             raise TypeError(f"'{val.__module__}.{cls_name}' must inherit from 'vedro.Scenario'")
 
-        # # If only the inheritance is correct, raise an error about incorrect naming convention
-        elif is_scenario_subclass:
-            raise ValueError(f"'{val.__module__}.{cls_name}' must have a name "
-                             "that starts or ends with 'Scenario'")
-
-        # If neither criteria are met, it's not a scenario
-        else:
-            return False
+        return False

vedro/core/scenario_loader/_scenario_loader.py

@@ -9,22 +9,19 @@
 
 class ScenarioLoader(ABC):
     """
-    Abstract base class for a scenario loader.
+    Represents an abstract base class for loading scenarios from a given path.
 
-    This class defines the interface for loading Vedro scenarios. Concrete implementations
-    of this class should provide the functionality to load scenarios from a specified source.
+    This class defines an interface for loading scenarios, which should be
+    implemented by subclasses to provide specific loading mechanisms.
     """
 
     @abstractmethod
     async def load(self, path: Path) -> List[Type[Scenario]]:
         """
-        Loads Vedro scenarios from a given path.
+        Load scenarios from the specified path.
 
-        This is an abstract method that must be implemented by subclasses. It should
-        define how scenarios are loaded from the specified path.
-
-        :param path: The file path or directory from which to load scenarios.
-        :return: A list of loaded Vedro scenario classes.
+        :param path: The path from which to load scenarios.
+        :return: A list of loaded scenarios.
         :raises NotImplementedError: This method should be overridden in subclasses.
         """
         pass