➕ pytest-cov is installed. 75% threshhold is set

Makefile

@@ -8,6 +8,7 @@ install.dev:
 	python -m pip install -e .[dev]
 
 
+
 .PHONY: build
 build:
 	python setup.py sdist bdist_wheel
@@ -35,7 +36,7 @@ quality:
 
 .PHONY: test
 test:
-	python -m pytest -s -vvv --cache-clear tests
+	python -m pytesttests
 
 
 .PHONY: test.unit

pyproject.toml

@@ -45,10 +45,11 @@ code_quality = [
     "isort",
     "mypy",
     "pytest",
+    "pytest-cov",
     "pytest-mock",
     "ruff",
     "tox",
-    "types-requests"
+    "types-requests",
 ]
 
 
@@ -94,8 +95,7 @@ lint.select = ["E", "F", "W"]
 
 
 [tool.pytest.ini_options]
-addopts = '-s -vvv --cache-clear'
-asyncio_mode = 'auto'
+addopts = '-s -vvv --cache-clear --cov-report=term-missing --cov --cov-fail-under=75'
 markers = [
     "smoke: quick tests to check basic functionality",
     "sanity: detailed tests to ensure major functions work correctly",

src/guidellm/backend/openai.py

@@ -1,6 +1,6 @@
 import functools
 import os
-from typing import Any, Dict, Iterator, List, Optional
+from typing import Any, Dict, Generator, List, Optional
 
 from loguru import logger
 from openai import OpenAI, Stream
@@ -72,7 +72,7 @@ def __init__(
 
     def make_request(
         self, request: TextGenerationRequest
-    ) -> Iterator[GenerativeResponse]:
+    ) -> Generator[GenerativeResponse, None, None]:
         """
         Make a request to the OpenAI backend.
 

src/guidellm/core/result.py

@@ -134,7 +134,7 @@ def start_time(self) -> float:
         :rtype: float
         """
 
-        self._recording_started()
+        self.check_recording_started()
         assert self._start_time
 
         return self._start_time
@@ -148,6 +148,7 @@ def end_time(self) -> float:
         :rtype: float
         """
 
+        self.check_recording_started()
         assert self._end_time
         return self._end_time
 
@@ -192,7 +193,7 @@ def start(self, prompt: str):
 
         logger.info(f"Text generation started with prompt: '{prompt}'")
 
-    def _recording_started(self, raise_exception: bool = True) -> bool:
+    def check_recording_started(self, raise_exception: bool = True) -> bool:
         """
         Ensure that the benchmark text generation recording is started.
 
@@ -205,7 +206,7 @@ def _recording_started(self, raise_exception: bool = True) -> bool:
         else:
             if raise_exception is True:
                 raise ValueError(
-                    "start time is not specified. "
+                    "Start time is not specified. "
                     "Did you make the `text_generation_benchmark.start()`?"
                 )
             else:
@@ -270,7 +271,11 @@ class TextGenerationError:
     :type error: Exception
     """
 
-    def __init__(self, request: TextGenerationRequest, error: Exception):
+    def __init__(
+        self,
+        request: TextGenerationRequest,
+        error_class: BaseException,
+    ):
         """
         Initialize the TextGenerationError with a unique identifier.
 
@@ -279,10 +284,10 @@ def __init__(self, request: TextGenerationRequest, error: Exception):
         :param error: The exception that occurred during the text generation.
         :type error: Exception
         """
-        self._request = request
-        self._error = error
+        self._request: TextGenerationRequest = request
+        self._error_class: BaseException = error_class
 
-        logger.error(f"Error occurred for request: {self._request}: {error}")
+        logger.error(f"Error occurred for request: {self._request}: {error_class}")
 
     def __repr__(self) -> str:
         """
@@ -291,7 +296,9 @@ def __repr__(self) -> str:
         :return: String representation of the TextGenerationError.
         :rtype: str
         """
-        return f"TextGenerationError(request={self._request}, error={self._error})"
+        return (
+            f"TextGenerationError(request={self._request}, error={self._error_class})"
+        )
 
     @property
     def request(self) -> TextGenerationRequest:
@@ -304,14 +311,14 @@ def request(self) -> TextGenerationRequest:
         return self._request
 
     @property
-    def error(self) -> Exception:
+    def error(self) -> BaseException:
         """
         Get the exception that occurred during the text generation.
 
         :return: The exception.
         :rtype: Exception
         """
-        return self._error
+        return self._error_class
 
 
 @dataclass
@@ -350,9 +357,11 @@ def __init__(self, mode: str, rate: Optional[float]):
         self._results: List[TextGenerationResult] = []
         self._errors: List[TextGenerationError] = []
         self._concurrencies: List[RequestConcurrencyMeasurement] = []
-        self._overloaded = False
         self._args_rate: Optional[float] = None
 
+        # NOTE: This state never changes
+        self._overloaded = False
+
         logger.debug(
             f"Initialized TextGenerationBenchmark with mode={mode} and rate={rate}"
         )
@@ -413,6 +422,7 @@ def overloaded(self) -> bool:
         :return: The overloaded state.
         :rtype: bool
         """
+
         return self._overloaded
 
     @property

src/guidellm/executor/__init__.py

@@ -1,15 +1,15 @@
 from .executor import Executor
 from .profile_generator import (
     Profile,
-    ProfileGenerationModes,
+    ProfileGenerationMode,
     ProfileGenerator,
     SingleProfileGenerator,
     SweepProfileGenerator,
 )
 
 __all__ = [
     "Executor",
-    "ProfileGenerationModes",
+    "ProfileGenerationMode",
     "Profile",
     "ProfileGenerator",
     "SingleProfileGenerator",

src/guidellm/executor/executor.py

@@ -1,39 +1,51 @@
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional
 
 from guidellm.backend import Backend
-from guidellm.core import TextGenerationBenchmarkReport
-from guidellm.executor.profile_generator import ProfileGenerationModes, ProfileGenerator
+from guidellm.core import TextGenerationBenchmark, TextGenerationBenchmarkReport
 from guidellm.request import RequestGenerator
-from guidellm.scheduler.scheduler import Scheduler
+from guidellm.scheduler import Scheduler
+
+from .profile_generator import ProfileGenerationMode, ProfileGenerator
 
 __all__ = ["Executor"]
 
 
 class Executor:
+    """
+    The main purpose of the `class Executor` is to dispatch running tasks according
+    to the Profile Generation mode
+    """
+
     def __init__(
         self,
-        request_generator: RequestGenerator,
         backend: Backend,
-        profile_mode: Union[str, ProfileGenerationModes] = "single",
+        request_generator: RequestGenerator,
+        profile_mode: ProfileGenerationMode = ProfileGenerationMode.SINGLE,
         profile_args: Optional[Dict[str, Any]] = None,
         max_requests: Optional[int] = None,
         max_duration: Optional[float] = None,
     ):
         self.request_generator = request_generator
         self.backend = backend
-        self.profile = ProfileGenerator.create_generator(
+        self.profile_generator: ProfileGenerator = ProfileGenerator.create(
             profile_mode, **(profile_args or {})
         )
-        self.max_requests = max_requests
-        self.max_duration = max_duration
+        self.max_requests: Optional[int] = max_requests
+        self.max_duration: Optional[float] = max_duration
+        self._scheduler: Optional[Scheduler] = None
+
+    @property
+    def scheduler(self) -> Scheduler:
+        if self._scheduler is None:
+            raise ValueError("The scheduler is not set. Did you run the execution?")
+        else:
+            return self._scheduler
 
     def run(self) -> TextGenerationBenchmarkReport:
         report = TextGenerationBenchmarkReport()
 
         while True:
-            profile = self.profile.next_profile(report)
-
-            if profile is None:
+            if not (profile := self.profile_generator.next(report)):
                 break
 
             scheduler = Scheduler(
@@ -45,7 +57,7 @@ def run(self) -> TextGenerationBenchmarkReport:
                 max_duration=self.max_duration,
             )
 
-            benchmark = scheduler.run()
+            benchmark: TextGenerationBenchmark = scheduler.run()
             report.add_benchmark(benchmark)
 
         return report