From c13b305172261b01d08c820823b1c3e418c0ded5 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Thu, 11 Jul 2024 11:18:19 -0400 Subject: [PATCH] Add serialization and deserialization classes and methods with pydantic --- src/guidellm/core/distribution.py | 111 ++---- src/guidellm/core/request.py | 117 +------ src/guidellm/core/result.py | 485 +++++---------------------- src/guidellm/core/serializable.py | 72 ++++ tests/unit/core/test_distribution.py | 60 +++- tests/unit/core/test_request.py | 52 ++- tests/unit/core/test_result.py | 41 +-- tests/unit/core/test_serializable.py | 40 +++ 8 files changed, 337 insertions(+), 641 deletions(-) create mode 100644 src/guidellm/core/serializable.py create mode 100644 tests/unit/core/test_serializable.py diff --git a/src/guidellm/core/distribution.py b/src/guidellm/core/distribution.py index 2a84ba2..f9c8926 100644 --- a/src/guidellm/core/distribution.py +++ b/src/guidellm/core/distribution.py @@ -1,76 +1,51 @@ -from typing import List, Union +from typing import List, Optional, Union import numpy as np from loguru import logger +from guidellm.core.serializable import Serializable + __all__ = ["Distribution"] -class Distribution: +class Distribution(Serializable): """ - A class to represent a statistical distribution and perform various statistical - analyses. - - :param data: List of numerical data points (int or float) to initialize the - distribution. - :type data: List[Union[int, float]], optional + A class to represent a statistical distribution and perform various + statistical analyses. """ - def __init__(self, data: List[Union[int, float]] = None): - """ - Initialize the Distribution with optional data. + data: Optional[List[Union[int, float]]] = [] - :param data: List of numerical data points to initialize the distribution, - defaults to None. - :type data: List[Union[int, float]], optional - """ - self._data = list(data) if data else [] - logger.debug(f"Initialized Distribution with data: {self._data}") + def __init__(self, **data): + super().__init__(**data) + logger.debug(f"Initialized Distribution with data: {self.data}") def __str__(self) -> str: """ Return a string representation of the Distribution. - - :return: String representation of the Distribution. - :rtype: str """ return ( f"Distribution(mean={self.mean:.2f}, median={self.median:.2f}, " - f"min={self.min}, max={self.max}, count={len(self._data)})" + f"min={self.min}, max={self.max}, count={len(self.data)})" ) def __repr__(self) -> str: """ Return an unambiguous string representation of the Distribution for debugging. - - :return: Unambiguous string representation of the Distribution. - :rtype: str """ - return f"Distribution(data={self._data})" - - @property - def data(self) -> List[Union[int, float]]: - """ - Return the data points of the distribution. - - :return: The data points of the distribution. - :rtype: List[Union[int, float]] - """ - return self._data + return f"Distribution(data={self.data})" @property def mean(self) -> float: """ Calculate and return the mean of the distribution. - :return: The mean of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate mean.") return 0.0 - mean_value = np.mean(self._data).item() + mean_value = np.mean(self.data).item() logger.debug(f"Calculated mean: {mean_value}") return mean_value @@ -78,15 +53,13 @@ def mean(self) -> float: def median(self) -> float: """ Calculate and return the median of the distribution. - :return: The median of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate median.") return 0.0 - median_value = np.median(self._data).item() + median_value = np.median(self.data).item() logger.debug(f"Calculated median: {median_value}") return median_value @@ -94,15 +67,13 @@ def median(self) -> float: def variance(self) -> float: """ Calculate and return the variance of the distribution. - :return: The variance of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate variance.") return 0.0 - variance_value = np.var(self._data).item() + variance_value = np.var(self.data).item() logger.debug(f"Calculated variance: {variance_value}") return variance_value @@ -110,49 +81,41 @@ def variance(self) -> float: def std_deviation(self) -> float: """ Calculate and return the standard deviation of the distribution. - :return: The standard deviation of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate standard deviation.") return 0.0 - std_deviation_value = np.std(self._data).item() + std_deviation_value = np.std(self.data).item() logger.debug(f"Calculated standard deviation: {std_deviation_value}") return std_deviation_value def percentile(self, percentile: float) -> float: """ Calculate and return the specified percentile of the distribution. - :param percentile: The desired percentile to calculate (0-100). - :type percentile: float :return: The specified percentile of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate percentile.") return 0.0 - percentile_value = np.percentile(self._data, percentile) + percentile_value = np.percentile(self.data, percentile) logger.debug(f"Calculated {percentile}th percentile: {percentile_value}") return percentile_value def percentiles(self, percentiles: List[float]) -> List[float]: """ Calculate and return the specified percentiles of the distribution. - :param percentiles: A list of desired percentiles to calculate (0-100). - :type percentiles: List[float] :return: A list of the specified percentiles of the distribution. - :rtype: List[float] """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate percentiles.") return [0.0] * len(percentiles) - percentiles_values = np.percentile(self._data, percentiles).tolist() + percentiles_values = np.percentile(self.data, percentiles).tolist() logger.debug(f"Calculated percentiles {percentiles}: {percentiles_values}") return percentiles_values @@ -160,15 +123,13 @@ def percentiles(self, percentiles: List[float]) -> List[float]: def min(self) -> float: """ Return the minimum value of the distribution. - :return: The minimum value of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate minimum.") return 0.0 - min_value = np.min(self._data) + min_value = np.min(self.data) logger.debug(f"Calculated min: {min_value}") return min_value @@ -176,15 +137,13 @@ def min(self) -> float: def max(self) -> float: """ Return the maximum value of the distribution. - :return: The maximum value of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate maximum.") return 0.0 - max_value = np.max(self._data) + max_value = np.max(self.data) logger.debug(f"Calculated max: {max_value}") return max_value @@ -192,11 +151,9 @@ def max(self) -> float: def range(self) -> float: """ Calculate and return the range of the distribution (max - min). - :return: The range of the distribution. - :rtype: float """ - if not self._data: + if not self.data: logger.warning("No data points available to calculate range.") return 0.0 @@ -207,9 +164,7 @@ def range(self) -> float: def describe(self) -> dict: """ Return a dictionary describing various statistics of the distribution. - :return: A dictionary with statistical summaries of the distribution. - :rtype: dict """ description = { "mean": self.mean, @@ -230,19 +185,15 @@ def describe(self) -> dict: def add_data(self, new_data: List[Union[int, float]]): """ Add new data points to the distribution. - :param new_data: A list of new numerical data points to add. - :type new_data: List[Union[int, float]] """ - self._data.extend(new_data) + self.data.extend(new_data) logger.debug(f"Added new data: {new_data}") def remove_data(self, remove_data: List[Union[int, float]]): """ Remove specified data points from the distribution. - :param remove_data: A list of numerical data points to remove. - :type remove_data: List[Union[int, float]] """ - self._data = [item for item in self._data if item not in remove_data] + self.data = [item for item in self.data if item not in remove_data] logger.debug(f"Removed data: {remove_data}") diff --git a/src/guidellm/core/request.py b/src/guidellm/core/request.py index 545c815..a0f6bd6 100644 --- a/src/guidellm/core/request.py +++ b/src/guidellm/core/request.py @@ -1,121 +1,34 @@ import uuid from typing import Any, Dict, Optional -from loguru import logger +from guidellm.core.serializable import Serializable __all__ = ["TextGenerationRequest"] -class TextGenerationRequest: +class TextGenerationRequest(Serializable): """ A class to represent a text generation request for generative AI workloads. - - :param prompt: The input prompt for the text generation request. - :type prompt: str - :param prompt_token_count: The number of tokens in the prompt, defaults to None. - :type prompt_token_count: Optional[int] - :param generated_token_count: The number of tokens to generate, defaults to None. - :type generated_token_count: Optional[int] - :param params: Optional parameters for the text generation request, - defaults to None. - :type params: Optional[Dict[str, Any]] """ + id: str + prompt: str + prompt_token_count: Optional[int] + generated_token_count: Optional[int] + params: Dict[str, Any] + def __init__( self, prompt: str, prompt_token_count: Optional[int] = None, generated_token_count: Optional[int] = None, params: Optional[Dict[str, Any]] = None, + id: Optional[str] = None, ): - """ - Initialize the TextGenerationRequest with a prompt and optional parameters. - - :param prompt: The input prompt for the text generation request. - :type prompt: str - :param prompt_token_count: The number of tokens in the prompt, defaults to None. - :type prompt_token_count: Optional[int] - :param generated_token_count: The number of tokens to generate, - defaults to None. - :type generated_token_count: Optional[int] - :param params: Optional parameters for the text generation request, - defaults to None. - :type params: Optional[Dict[str, Any]] - """ - self._id = str(uuid.uuid4()) - self._prompt = prompt - self._prompt_token_count = prompt_token_count - self._generated_token_count = generated_token_count - self._params = params or {} - - logger.debug( - f"Initialized TextGenerationRequest with id={self._id}, " - f"prompt={prompt}, prompt_token_count={prompt_token_count}, " - f"generated_token_count={generated_token_count}, params={params}" + super().__init__( + id=str(uuid.uuid4()) if id is None else id, + prompt=prompt, + prompt_token_count=prompt_token_count, + generated_token_count=generated_token_count, + params=params or {}, ) - - def __repr__(self) -> str: - """ - Return a string representation of the TextGenerationRequest. - - :return: String representation of the TextGenerationRequest. - :rtype: str - """ - return ( - f"TextGenerationRequest(" - f"id={self._id}, " - f"prompt={self._prompt}, " - f"prompt_token_count={self._prompt_token_count}, " - f"generated_token_count={self._generated_token_count}, " - f"params={self._params})" - ) - - @property - def id(self) -> str: - """ - Get the unique identifier for the text generation request. - - :return: The unique identifier. - :rtype: str - """ - return self._id - - @property - def prompt(self) -> str: - """ - Get the input prompt for the text generation request. - - :return: The input prompt. - :rtype: str - """ - return self._prompt - - @property - def prompt_token_count(self) -> Optional[int]: - """ - Get the number of tokens in the prompt for the text generation request. - - :return: The number of tokens in the prompt. - :rtype: Optional[int] - """ - return self._prompt_token_count - - @property - def generated_token_count(self) -> Optional[int]: - """ - Get the number of tokens to generate for the text generation request. - - :return: The number of tokens to generate. - :rtype: Optional[int] - """ - return self._generated_token_count - - @property - def params(self) -> Dict[str, Any]: - """ - Get the optional parameters for the text generation request. - - :return: The optional parameters. - :rtype: Dict[str, Any] - """ - return self._params diff --git a/src/guidellm/core/result.py b/src/guidellm/core/result.py index a72a03b..07ef628 100644 --- a/src/guidellm/core/result.py +++ b/src/guidellm/core/result.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from time import perf_counter, time from typing import Any, Dict, List, Optional, Union @@ -6,6 +5,7 @@ from guidellm.core.distribution import Distribution from guidellm.core.request import TextGenerationRequest +from guidellm.core.serializable import Serializable __all__ = [ "TextGenerationResult", @@ -16,148 +16,25 @@ ] -class TextGenerationResult: +class TextGenerationResult(Serializable): """ A class to represent the result of a text generation request for generative AI workloads. - - :param request: The text generation request that generated this result. - :type request: TextGenerationRequest """ - def __init__(self, request: TextGenerationRequest): - """ - Initialize the TextGenerationResult with the given text generation request. - - :param request: The text generation request that generated this result. - :type request: TextGenerationRequest - """ - self._request = request - self._prompt = "" - self._prompt_word_count = 0 - self._prompt_token_count = 0 - self._output = "" - self._output_word_count = 0 - self._output_token_count = 0 - self._last_time: Optional[float] = None - self._first_token_set: bool = False - self._start_time: Optional[float] = None - self._end_time: Optional[float] = None - self._first_token_time: Optional[float] = None - self._decode_times = Distribution() - - logger.debug(f"Initialized TextGenerationResult for request: {self._request}") - - def __repr__(self) -> str: - return ( - f"TextGenerationResult(" - f"request_id={self._request.id}, " - f"prompt='{self._prompt}', " - f"output='{self._output}', " - f"start_time={self._start_time}, " - f"end_time={self._end_time}, " - f"first_token_time={self._first_token_time}, " - f"decode_times={self._decode_times})" - ) - - def __str__(self) -> str: - return ( - f"TextGenerationResult(" - f"request_id={self._request.id}, " - f"prompt='{self._prompt}', " - f"output='{self._output}', " - f"start_time={self._start_time}, " - f"end_time={self._end_time})" - ) - - def __eq__(self, other: "TextGenerationResult") -> bool: - """ - Check equality between two TextGenerationResult instances. - - :param other: Another instance of TextGenerationResult. - :type other: TextGenerationResult - :return: True if the instances are equal, False otherwise. - :rtype: bool - """ - return ( - self._request == other._request - and self._prompt == other._prompt - and self._output == other._output - and self._start_time == other._start_time - and self._end_time == other._end_time - and self._first_token_time == other._first_token_time - and self._decode_times == other._decode_times - ) - - @property - def request(self) -> TextGenerationRequest: - """ - Get the text generation request associated with this result. - - :return: The text generation request. - :rtype: TextGenerationRequest - """ - return self._request - - @property - def prompt(self) -> str: - """ - Get the prompt used in the text generation. - - :return: The prompt. - :rtype: str - """ - return self._prompt - - @property - def output(self) -> str: - """ - Get the generated output from the text generation. - - :return: The generated output. - :rtype: str - """ - return self._output - - @property - def start_time(self) -> Optional[float]: - """ - Get the start time of the text generation. - - :return: The start time. - :rtype: Optional[float] - """ - return self._start_time - - @property - def end_time(self) -> Optional[float]: - """ - Get the end time of the text generation. - - :return: The end time. - :rtype: Optional[float] - """ - return self._end_time - - @property - def first_token_time(self) -> Optional[float]: - """ - Get the time taken to generate the first token. - - :return: The time taken to generate the first token. - :rtype: Optional[float] - """ - return self._first_token_time - - @property - def decode_times(self) -> Distribution: - """ - Get the decode times for each token in the text generation. - - :return: The decode times. - :rtype: Distribution - """ - return self._decode_times + request: TextGenerationRequest + prompt: str = "" + prompt_word_count: int = 0 + prompt_token_count: int = 0 + output: str = "" + output_word_count: int = 0 + output_token_count: int = 0 + last_time: Optional[float] = None + first_token_set: bool = False + start_time: Optional[float] = None + end_time: Optional[float] = None + first_token_time: Optional[float] = None + decode_times: Distribution = Distribution() def start(self, prompt: str): """ @@ -166,14 +43,14 @@ def start(self, prompt: str): :param prompt: The input prompt for the text generation. :type prompt: str """ - self._prompt = prompt - self._prompt_word_count = len(prompt.split()) - self._prompt_token_count = len(prompt) # Token count placeholder - self._start_time = time() - self._last_time = perf_counter() - self._first_token_set = False + self.prompt = prompt + self.prompt_word_count = len(prompt.split()) + self.prompt_token_count = len(prompt) # Token count placeholder + self.start_time = time() + self.last_time = perf_counter() + self.first_token_set = False - logger.info(f"Text generation started with prompt: '{prompt}'") + logger.info("Text generation started with prompt: '{}'", prompt) def output_token(self, token: str): """ @@ -184,17 +61,18 @@ def output_token(self, token: str): """ current_counter = perf_counter() - if not self._first_token_set: - self._first_token_time = current_counter - self._last_time - self._first_token_set = True - logger.debug(f"First token decode time: {self._first_token_time}") + if not self.first_token_set: + self.first_token_time = current_counter - self.last_time + self.first_token_set = True + logger.debug(f"First token decode time: {self.first_token_time}") else: - decode_time = current_counter - self._last_time - self._decode_times.add_data([decode_time]) + decode_time = current_counter - self.last_time + self.decode_times.add_data([decode_time]) logger.debug(f"Token '{token}' decoded in {decode_time} seconds") - self._last_time = current_counter - self._output += f"{token} " + self.last_time = current_counter + self.output += f"{token} " + logger.debug("Added token {} to output", token) def end( self, @@ -214,91 +92,40 @@ def end( defaults to word count. :type output_token_count: Optional[int] """ - self._output = output - self._end_time = time() - self._output_word_count = len(output.split()) - self._output_token_count = ( + self.output = output + self.end_time = time() + self.output_word_count = len(output.split()) + self.output_token_count = ( output_token_count if output_token_count is not None - else self._output_word_count + else self.output_word_count ) - self._prompt_token_count = ( + self.prompt_token_count = ( prompt_token_count if prompt_token_count is not None - else self._prompt_word_count + else self.prompt_word_count ) - logger.info(f"Text generation ended with output: '{output}'") + logger.info("Text generation ended with output: '{}'", output) -class TextGenerationError: +class TextGenerationError(Serializable): """ A class to represent an error that occurred during a text generation request for generative AI workloads. - - :param request: The text generation request that generated this error. - :type request: TextGenerationRequest - :param error: The exception that occurred during the text generation. - :type error: Exception """ - def __init__(self, request: TextGenerationRequest, error: Exception): - """ - Initialize the TextGenerationError with a unique identifier. - - :param request: The text generation request that generated this error. - :type request: TextGenerationRequest - :param error: The exception that occurred during the text generation. - :type error: Exception - """ - self._request = request - self._error = error - - logger.error(f"Error occurred for request: {self._request}: {error}") - - def __repr__(self) -> str: - """ - Return a string representation of the TextGenerationError. - - :return: String representation of the TextGenerationError. - :rtype: str - """ - return f"TextGenerationError(request={self._request}, error={self._error})" - - @property - def request(self) -> TextGenerationRequest: - """ - Get the text generation request associated with this error. - - :return: The text generation request. - :rtype: TextGenerationRequest - """ - return self._request - - @property - def error(self) -> Exception: - """ - Get the exception that occurred during the text generation. + request: TextGenerationRequest + error: str - :return: The exception. - :rtype: Exception - """ - return self._error + def __init__(self, request: TextGenerationRequest, error: Exception): + super().__init__(request=request, error=str(error)) + logger.error("Text generation error occurred: {}", error) -@dataclass -class RequestConcurrencyMeasurement: +class RequestConcurrencyMeasurement(Serializable): """ A dataclass to represent the concurrency measurement of a request. - - :param time: The time at which the measurement was taken. - :type time: float - :param completed: The number of completed requests. - :type completed: int - :param errored: The number of errored requests. - :type errored: int - :param processing: The number of requests currently being processed. - :type processing: int """ time: float @@ -307,62 +134,18 @@ class RequestConcurrencyMeasurement: processing: int -class TextGenerationBenchmark: - def __init__(self, mode: str, rate: Optional[float]): - """ - Initialize the TextGenerationBenchmark. - - :param mode: The mode of the result. - :type mode: str - :param rate: The rate of requests. - :type rate: Optional[float] - """ - self._mode = mode - self._rate = rate - self._results: List[TextGenerationResult] = [] - self._errors: List[TextGenerationError] = [] - self._concurrencies: List[RequestConcurrencyMeasurement] = [] - - logger.debug( - f"Initialized TextGenerationBenchmark with mode={mode} and rate={rate}" - ) - - def __repr__(self) -> str: - return ( - f"TextGenerationBenchmark(" - f"mode={self._mode}, " - f"rate={self._rate}, " - f"results={self._results}, " - f"errors={self._errors}, " - f"concurrencies={self._concurrencies})" - ) - - def __str__(self) -> str: - return ( - f"TextGenerationBenchmark(" - f"mode={self._mode}, " - f"rate={self._rate}, " - f"request_count={self.request_count}, " - f"error_count={self.error_count}, " - f"request_rate={self.request_rate})" - ) - - def __eq__(self, other: "TextGenerationBenchmark") -> bool: - """ - Check equality between two TextGenerationBenchmark instances. +class TextGenerationBenchmark(Serializable): + """ + A class to represent a benchmark of text generation requests + (results and errors) for generative AI workloads. + This is a set of results and errors for a specific mode and rate. + """ - :param other: Another instance of TextGenerationBenchmark. - :type other: TextGenerationBenchmark - :return: True if the instances are equal, False otherwise. - :rtype: bool - """ - return ( - self._mode == other._mode - and self._rate == other._rate - and self._results == other._results - and self._errors == other._errors - and self._concurrencies == other._concurrencies - ) + mode: str + rate: Optional[float] + results: List[TextGenerationResult] = [] + errors: List[TextGenerationError] = [] + concurrencies: List[RequestConcurrencyMeasurement] = [] def __iter__(self): """ @@ -370,57 +153,7 @@ def __iter__(self): :return: An iterator over the results. """ - return iter(self._results) - - @property - def mode(self) -> str: - """ - Get the mode of the result. - - :return: The mode. - :rtype: str - """ - return self._mode - - @property - def rate(self) -> Optional[float]: - """ - Get the rate of requests in the result. - - :return: The rate of requests. - :rtype: Optional[float] - """ - return self._rate - - @property - def results(self) -> List[TextGenerationResult]: - """ - Get the list of results in the result. - - :return: The list of results. - :rtype: List[TextGenerationResult] - """ - return self._results - - @property - def errors(self) -> List[TextGenerationError]: - """ - Get the list of errors in the result. - - :return: The list of errors. - :rtype: List[TextGenerationError] - """ - return self._errors - - @property - def concurrencies(self) -> List[RequestConcurrencyMeasurement]: - """ - Get the list of concurrency measurements in the result. - - :return: The list of concurrency measurements. - :rtype: List[RequestConcurrencyMeasurement] - """ - return self._concurrencies + return iter(self.results) @property def request_count(self) -> int: @@ -430,7 +163,7 @@ def request_count(self) -> int: :return: The number of requests. :rtype: int """ - return len(self._results) + return len(self.results) @property def error_count(self) -> int: @@ -440,7 +173,7 @@ def error_count(self) -> int: :return: The number of errors. :rtype: int """ - return len(self._errors) + return len(self.errors) @property def request_rate(self) -> float: @@ -450,11 +183,11 @@ def request_rate(self) -> float: :return: The rate of requests per second. :rtype: float """ - if not self._results: + if not self.results: return 0.0 - start_time = self._results[0].start_time - end_time = self._results[-1].end_time + start_time = self.results[0].start_time + end_time = self.results[-1].end_time return self.request_count / (end_time - start_time) @@ -462,15 +195,15 @@ def request_started(self): """ Record the start of a generation request. """ - if not self._concurrencies: - self._concurrencies.append( + if not self.concurrencies: + self.concurrencies.append( RequestConcurrencyMeasurement( time=time(), completed=0, errored=0, processing=1 ) ) else: - last = self._concurrencies[-1] - self._concurrencies.append( + last = self.concurrencies[-1] + self.concurrencies.append( RequestConcurrencyMeasurement( time=time(), completed=last.completed, @@ -491,9 +224,9 @@ def request_completed( :type result: Union[TextGenerationResult, TextGenerationError] """ if isinstance(result, TextGenerationError): - self._errors.append(result) - last = self._concurrencies[-1] - self._concurrencies.append( + self.errors.append(result) + last = self.concurrencies[-1] + self.concurrencies.append( RequestConcurrencyMeasurement( time=time(), completed=last.completed, @@ -501,11 +234,13 @@ def request_completed( processing=last.processing - 1, ) ) - logger.info(f"Text generation request resulted in error: {result}") + logger.warning( + "Text generation request resulted in error: {}", result.error + ) else: - self._results.append(result) - last = self._concurrencies[-1] - self._concurrencies.append( + self.results.append(result) + last = self.concurrencies[-1] + self.concurrencies.append( RequestConcurrencyMeasurement( time=time(), completed=last.completed + 1, @@ -513,71 +248,21 @@ def request_completed( processing=last.processing - 1, ) ) - logger.info(f"Text generation request completed successfully: {result}") + logger.info("Text generation request completed successfully: {}", result) -class TextGenerationBenchmarkReport: +class TextGenerationBenchmarkReport(Serializable): """ A class to represent a report of text generation benchmarks for generative AI workloads. + This is a collection of benchmarks for different modes and rates. """ - def __init__(self): - """ - Initialize the TextGenerationBenchmarkReport. - """ - self._benchmarks: List[TextGenerationBenchmark] = [] - self._args: List[Dict[str, Any]] = [] - - logger.debug("Initialized TextGenerationBenchmarkReport") - - def __repr__(self) -> str: - return ( - f"TextGenerationBenchmarkReport(" - f"benchmarks={self._benchmarks}, " - f"args={self._args})" - ) - - def __str__(self) -> str: - return ( - f"TextGenerationBenchmarkReport(" - f"args={self._args}, " - f"benchmarks_summary=[{', '.join(str(b) for b in self._benchmarks)}])" - ) - - def __eq__(self, other: "TextGenerationBenchmarkReport") -> bool: - """ - Check equality between two TextGenerationBenchmarkReport instances. - - :param other: Another instance of TextGenerationBenchmarkReport. - :type other: TextGenerationBenchmarkReport - :return: True if the instances are equal, False otherwise. - :rtype: bool - """ - return self._benchmarks == other._benchmarks and self._args == other._args + benchmarks: List[TextGenerationBenchmark] = [] + args: List[Dict[str, Any]] = [] def __iter__(self): - return iter(self._benchmarks) - - @property - def benchmarks(self) -> List[TextGenerationBenchmark]: - """ - Get the list of benchmarks. - - :return: The list of benchmarks. - :rtype: List[TextGenerationBenchmark] - """ - return self._benchmarks - - @property - def args(self) -> List[Dict[str, Any]]: - """ - Get the list of arguments. - - :return: The list of arguments. - :rtype: List[Dict[str, Any]] - """ - return self._args + return iter(self.benchmarks) @property def benchmarks_sorted(self) -> List[TextGenerationBenchmark]: @@ -587,7 +272,7 @@ def benchmarks_sorted(self) -> List[TextGenerationBenchmark]: :return: The sorted list of benchmarks. :rtype: List[TextGenerationBenchmark] """ - benchmarks = sorted(self._benchmarks, key=lambda x: x.request_rate) + benchmarks = sorted(self.benchmarks, key=lambda x: x.request_rate) return benchmarks def add_benchmark(self, benchmark: TextGenerationBenchmark): @@ -597,5 +282,5 @@ def add_benchmark(self, benchmark: TextGenerationBenchmark): :param benchmark: The result to add. :type benchmark: TextGenerationBenchmark """ - self._benchmarks.append(benchmark) - logger.debug(f"Added result: {benchmark}") + self.benchmarks.append(benchmark) + logger.debug("Added result: {}", benchmark) diff --git a/src/guidellm/core/serializable.py b/src/guidellm/core/serializable.py new file mode 100644 index 0000000..afa40dd --- /dev/null +++ b/src/guidellm/core/serializable.py @@ -0,0 +1,72 @@ +from typing import Any + +import yaml +from loguru import logger +from pydantic import BaseModel + + +class Serializable(BaseModel): + """ + A base class for models that require YAML and JSON serialization and + deserialization. + """ + + def __init__(self, /, **data: Any) -> None: + super().__init__(**data) + logger.debug( + "Initialized new instance of {} with data: {}", + self.__class__.__name__, + data, + ) + + def to_yaml(self) -> str: + """ + Serialize the model to a YAML string. + + :return: YAML string representation of the model. + """ + logger.debug("Serializing to YAML... {}", self) + yaml_str = yaml.dump(self.model_dump()) + logger.debug("Serialized to YAML: {}", yaml_str) + + return yaml_str + + @classmethod + def from_yaml(cls, data: str): + """ + Deserialize a YAML string to a model instance. + + :param data: YAML string to deserialize. + :return: An instance of the model. + """ + logger.debug("Deserializing from YAML... {}", data) + obj = cls.model_validate(yaml.safe_load(data)) + logger.debug("Deserialized from YAML: {}", obj) + + return obj + + def to_json(self) -> str: + """ + Serialize the model to a JSON string. + + :return: JSON string representation of the model. + """ + logger.debug("Serializing to JSON... {}", self) + json_str = self.model_dump_json() + logger.debug("Serialized to JSON: {}", json_str) + + return json_str + + @classmethod + def from_json(cls, data: str): + """ + Deserialize a JSON string to a model instance. + + :param data: JSON string to deserialize. + :return: An instance of the model. + """ + logger.debug("Deserializing from JSON... {}", data) + obj = cls.model_validate_json(data) + logger.debug("Deserialized from JSON: {}", obj) + + return obj diff --git a/tests/unit/core/test_distribution.py b/tests/unit/core/test_distribution.py index 87e82cf..40c7ced 100644 --- a/tests/unit/core/test_distribution.py +++ b/tests/unit/core/test_distribution.py @@ -6,14 +6,14 @@ @pytest.mark.smoke def test_distribution_initialization(): data = [1, 2, 3, 4, 5] - dist = Distribution(data) + dist = Distribution(data=data) assert dist.data == data -@pytest.mark.sanity +@pytest.mark.smoke def test_distribution_statistics(): data = [1, 2, 3, 4, 5] - dist = Distribution(data) + dist = Distribution(data=data) assert dist.mean == 3.0 assert dist.median == 3.0 assert dist.variance == 2.0 @@ -23,34 +23,64 @@ def test_distribution_statistics(): assert dist.range == 4 +@pytest.mark.sanity +def test_distribution_add_data(): + data = [1, 2, 3, 4, 5] + dist = Distribution(data=data) + new_data = [6, 7, 8] + dist.add_data(new_data) + + assert dist.data == data + new_data + + +@pytest.mark.sanity +def test_distribution_remove_data(): + data = [1, 2, 3, 4, 5] + dist = Distribution(data=data) + remove_data = [2, 4] + dist.remove_data(remove_data) + assert dist.data == [1, 3, 5] + + @pytest.mark.regression def test_distribution_str(): data = [1, 2, 3, 4, 5] - dist = Distribution(data) + dist = Distribution(data=data) assert str(dist) == "Distribution(mean=3.00, median=3.00, min=1, max=5, count=5)" @pytest.mark.regression def test_distribution_repr(): data = [1, 2, 3, 4, 5] - dist = Distribution(data) + dist = Distribution(data=data) assert repr(dist) == f"Distribution(data={data})" @pytest.mark.regression -def test_distribution_add_data(): +def test_distribution_to_json(): data = [1, 2, 3, 4, 5] - dist = Distribution(data) - new_data = [6, 7, 8] - dist.add_data(new_data) + dist = Distribution(data=data) + json_str = dist.to_json() + assert '"data":[1,2,3,4,5]' in json_str - assert dist.data == data + new_data + +@pytest.mark.regression +def test_distribution_from_json(): + json_str = '{"data": [1, 2, 3, 4, 5]}' + dist = Distribution.from_json(json_str) + assert dist.data == [1, 2, 3, 4, 5] @pytest.mark.regression -def test_distribution_remove_data(): +def test_distribution_to_yaml(): data = [1, 2, 3, 4, 5] - dist = Distribution(data) - remove_data = [2, 4] - dist.remove_data(remove_data) - assert dist.data == [1, 3, 5] + dist = Distribution(data=data) + yaml_str = dist.to_yaml() + assert "data:\n- 1\n- 2\n- 3\n- 4\n- 5\n" in yaml_str + + +@pytest.mark.regression +def test_distribution_from_yaml(): + yaml_str = "data:\n- 1\n- 2\n- 3\n- 4\n- 5\n" + dist = Distribution.from_yaml(yaml_str) + assert dist.data == [1, 2, 3, 4, 5] diff --git a/tests/unit/core/test_request.py b/tests/unit/core/test_request.py index 2b066d4..136a582 100644 --- a/tests/unit/core/test_request.py +++ b/tests/unit/core/test_request.py @@ -29,16 +29,46 @@ def test_text_generation_request_initialization_with_params(): @pytest.mark.regression -def test_text_generation_request_repr(): - prompt = "Generate a story" - prompt_token_count = 50 - generated_token_count = 100 - params = {"temperature": 0.7} - request = TextGenerationRequest( - prompt, prompt_token_count, generated_token_count, params +def test_request_to_json(): + prompt = "Generate text" + request = TextGenerationRequest(prompt=prompt) + json_str = request.to_json() + assert '"prompt":"Generate text"' in json_str + assert '"id":' in json_str + + +@pytest.mark.regression +def test_request_from_json(): + json_str = ( + '{"id": "12345", "prompt": "Generate text", "prompt_token_count": 10, ' + '"generated_token_count": 50, "params": {"temperature": 0.7}}' ) - assert repr(request) == ( - f"TextGenerationRequest(id={request.id}, prompt={prompt}, " - f"prompt_token_count={prompt_token_count}, " - f"generated_token_count={generated_token_count}, params={params})" + request = TextGenerationRequest.from_json(json_str) + assert request.id == "12345" + assert request.prompt == "Generate text" + assert request.prompt_token_count == 10 + assert request.generated_token_count == 50 + assert request.params == {"temperature": 0.7} + + +@pytest.mark.regression +def test_request_to_yaml(): + prompt = "Generate text" + request = TextGenerationRequest(prompt=prompt) + yaml_str = request.to_yaml() + assert "prompt: Generate text" in yaml_str + assert "id:" in yaml_str + + +@pytest.mark.regression +def test_request_from_yaml(): + yaml_str = ( + "id: '12345'\nprompt: Generate text\nprompt_token_count: 10\n" + "generated_token_count: 50\nparams:\n temperature: 0.7\n" ) + request = TextGenerationRequest.from_yaml(yaml_str) + assert request.id == "12345" + assert request.prompt == "Generate text" + assert request.prompt_token_count == 10 + assert request.generated_token_count == 50 + assert request.params == {"temperature": 0.7} diff --git a/tests/unit/core/test_result.py b/tests/unit/core/test_result.py index 8cedc59..081fa48 100644 --- a/tests/unit/core/test_result.py +++ b/tests/unit/core/test_result.py @@ -12,7 +12,7 @@ @pytest.mark.smoke def test_text_generation_result_initialization(): request = TextGenerationRequest(prompt="Generate a story") - result = TextGenerationResult(request) + result = TextGenerationResult(request=request) assert result.request == request assert result.prompt == "" assert result.output == "" @@ -21,34 +21,17 @@ def test_text_generation_result_initialization(): @pytest.mark.sanity def test_text_generation_result_start(): request = TextGenerationRequest(prompt="Generate a story") - result = TextGenerationResult(request) + result = TextGenerationResult(request=request) prompt = "Once upon a time" result.start(prompt) assert result.prompt == prompt assert result.start_time is not None -@pytest.mark.regression -def test_text_generation_result_repr(): - request = TextGenerationRequest(prompt="Generate a story") - result = TextGenerationResult(request) - - assert repr(result) == ( - f"TextGenerationResult(" - f"request_id={request.id}, " - f"prompt='', " - f"output='', " - f"start_time=None, " - f"end_time=None, " - f"first_token_time=None, " - f"decode_times=Distribution(mean=0.00, median=0.00, min=0.0, max=0.0, count=0))" - ) - - @pytest.mark.sanity def test_text_generation_result_end(): request = TextGenerationRequest(prompt="Generate a story") - result = TextGenerationResult(request) + result = TextGenerationResult(request=request) result.end("The end") assert result.output == "The end" assert result.end_time is not None @@ -58,17 +41,9 @@ def test_text_generation_result_end(): def test_text_generation_error_initialization(): request = TextGenerationRequest(prompt="Generate a story") error = Exception("Test error") - result = TextGenerationError(request, error) + result = TextGenerationError(request=request, error=error) assert result.request == request - assert result.error == error - - -@pytest.mark.regression -def test_text_generation_error_repr(): - request = TextGenerationRequest(prompt="Generate a story") - error = Exception("Test error") - result = TextGenerationError(request, error) - assert repr(result) == f"TextGenerationError(request={request}, error={error})" + assert result.error == str(error) @pytest.mark.smoke @@ -84,7 +59,7 @@ def test_text_generation_benchmark_initialization(): def test_text_generation_benchmark_started(): benchmark = TextGenerationBenchmark(mode="test", rate=1.0) benchmark.request_started() - assert len(benchmark._concurrencies) == 1 + assert len(benchmark.concurrencies) == 1 @pytest.mark.regression @@ -92,7 +67,7 @@ def test_text_generation_benchmark_completed_with_result(): benchmark = TextGenerationBenchmark(mode="test", rate=1.0) benchmark.request_started() request = TextGenerationRequest(prompt="Generate a story") - result = TextGenerationResult(request) + result = TextGenerationResult(request=request) benchmark.request_completed(result) assert benchmark.request_count == 1 assert benchmark.error_count == 0 @@ -103,7 +78,7 @@ def test_text_generation_benchmark_completed_with_error(): benchmark = TextGenerationBenchmark(mode="test", rate=1.0) benchmark.request_started() request = TextGenerationRequest(prompt="Generate a story") - error = TextGenerationError(request, Exception("Test error")) + error = TextGenerationError(request=request, error=Exception("Test error")) benchmark.request_completed(error) assert benchmark.request_count == 0 assert benchmark.error_count == 1 diff --git a/tests/unit/core/test_serializable.py b/tests/unit/core/test_serializable.py new file mode 100644 index 0000000..bd233e2 --- /dev/null +++ b/tests/unit/core/test_serializable.py @@ -0,0 +1,40 @@ +import pytest + +from guidellm.core.serializable import Serializable + + +class ExampleModel(Serializable): + name: str + age: int + + +@pytest.mark.smoke +def test_serializable_to_json(): + example = ExampleModel(name="John Doe", age=30) + json_str = example.to_json() + assert '"name":"John Doe"' in json_str + assert '"age":30' in json_str + + +@pytest.mark.smoke +def test_serializable_from_json(): + json_str = '{"name": "John Doe", "age": 30}' + example = ExampleModel.from_json(json_str) + assert example.name == "John Doe" + assert example.age == 30 + + +@pytest.mark.smoke +def test_serializable_to_yaml(): + example = ExampleModel(name="John Doe", age=30) + yaml_str = example.to_yaml() + assert "name: John Doe" in yaml_str + assert "age: 30" in yaml_str + + +@pytest.mark.smoke +def test_serializable_from_yaml(): + yaml_str = "name: John Doe\nage: 30\n" + example = ExampleModel.from_yaml(yaml_str) + assert example.name == "John Doe" + assert example.age == 30