From 98f79c48e0a4016bc5b21afa08b3f0955600b035 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 9 Jul 2024 18:28:17 -0400 Subject: [PATCH] update fixed rate profile generator, add tests, set up initial executor test --- src/guidellm/executor/profile_generator.py | 21 +++++--- tests/unit/executor/test_executor.py | 17 ++++++ tests/unit/executor/test_profile_generator.py | 54 ++++++++++++++----- 3 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 tests/unit/executor/test_executor.py diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 0fb206b..a9ef77c 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -69,30 +69,37 @@ def next_profile( @ProfileGenerator.register_generator(ProfileGenerationModes.FIXED) class FixedRateProfileGenerator(ProfileGenerator): - def __init__(self, rate: List[float], rate_type: str, **kwargs): + def __init__(self, rate_type: str, rate: Optional[List[float]] = None, **kwargs): super().__init__(ProfileGenerationModes.FIXED) - if rate_type == "synchronous" and len(rate) > 0: + if rate_type == "synchronous" and rate and len(rate) > 0: raise ValueError("custom rates are not supported in synchronous mode") self._rates = rate self._rate_index = 0 + self._generated = False self._rate_type = rate_type def next_profile( self, current_report: TextGenerationBenchmarkReport ) -> Optional[Profile]: - if self._rate_index >= len(self._rates): - return None + if self._rate_type == "synchronous": + if self._generated: + return None - current_rate = self._rates[self._rate_index] - self._rate_index += 1 + self._generated = True - if self._rate_type == "synchronous": return Profile( load_gen_mode=LoadGenerationModes.SYNCHRONOUS, load_gen_rate=None ) if self._rate_type in {"constant", "poisson"}: + if self._rate_index >= len(self._rates): + return None + + current_rate = self._rates[self._rate_index] + self._rate_index += 1 + load_gen_mode = RateTypeLoadGenModeMap[self._rate_type] + return Profile( load_gen_mode=load_gen_mode, load_gen_rate=current_rate ) diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py new file mode 100644 index 0000000..095f5ac --- /dev/null +++ b/tests/unit/executor/test_executor.py @@ -0,0 +1,17 @@ +from unittest.mock import MagicMock +from src.guidellm.backend.base import Backend +from src.guidellm.executor.executor import Executor +from src.guidellm.request.base import RequestGenerator + +def test_executor_creation(): + mock_request_generator = MagicMock(spec=RequestGenerator) + mock_backend = MagicMock(spec=Backend) + rate_type = "sweep" + profile_args = None + max_requests = None, + max_duration = None, + executor = Executor(mock_request_generator, mock_backend, rate_type, profile_args, max_requests, max_duration); + assert executor.request_generator == mock_request_generator + assert executor.backend == mock_backend + assert executor.max_requests == max_requests + assert executor.max_duration == max_duration diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index af69a3f..1c9f084 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -1,6 +1,8 @@ import pytest - +from unittest.mock import MagicMock from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) +from src.guidellm.core.result import TextGenerationBenchmarkReport +from src.guidellm.scheduler.load_generator import LoadGenerationModes def test_invalid_profile_generation_mode_error(): rate = [1] @@ -10,19 +12,45 @@ def test_invalid_profile_generation_mode_error(): ProfileGenerator.create_generator(profile_mode, **({ "rate": rate, "rate_type": rate_type})) def test_sweep_profile_generator_creation(): - profile = ProfileGenerator.create_generator("sweep", **({})) - assert isinstance(profile, SweepProfileGenerator) - assert profile._sync_run == False - assert profile._max_found == False - assert profile._pending_rates == None - assert profile._pending_rates == None + profile_generator = ProfileGenerator.create_generator("sweep", **({})) + assert isinstance(profile_generator, SweepProfileGenerator) + assert profile_generator._sync_run == False + assert profile_generator._max_found == False + assert profile_generator._pending_rates == None + assert profile_generator._pending_rates == None def test_fixed_rate_profile_generator_creation(): rate = [1] rate_type = "constant" - profile = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) - assert isinstance(profile, FixedRateProfileGenerator) - assert profile._rates == rate - assert profile._rate_type == rate_type - assert profile._rate_index == 0 - assert profile._rate_index == 0 \ No newline at end of file + profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + assert isinstance(profile_generator, FixedRateProfileGenerator) + assert profile_generator._rates == rate + assert profile_generator._rate_type == rate_type + assert profile_generator._rate_index == 0 + assert profile_generator._rate_index == 0 + +def test_synchronous_mode_rate_list_error(): + rate = [1] + rate_type = "synchronous" + with pytest.raises(ValueError, match="custom rates are not supported in synchronous mode"): + ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + +def test_next_profile_with_multiple_rates(): + rates = [1, 2] + rate_type = "constant" + profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rates, "rate_type": rate_type})) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + for rate in rates: + current_profile = profile_generator.next_profile(mock_report) + assert current_profile.load_gen_rate == rate + assert current_profile.load_gen_mode.name == LoadGenerationModes.CONSTANT.name + assert profile_generator.next_profile(mock_report) == None + +def test_next_profile_with_sync_mode(): + rate_type = "synchronous" + profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate_type": rate_type})) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + current_profile = profile_generator.next_profile(mock_report) + assert current_profile.load_gen_rate == None + assert current_profile.load_gen_mode.name == LoadGenerationModes.SYNCHRONOUS.name + assert profile_generator.next_profile(mock_report) == None \ No newline at end of file