From 3daba29a8a302ec08c32f3bcb9cff7b1c04cec8e Mon Sep 17 00:00:00 2001 From: Brady Planden <55357039+BradyPlanden@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:16:16 +0000 Subject: [PATCH] Refactor optimisation args (#551) * refactor: moves common Pints' optimisation keys from kwargs to optional args, aligns maxiter and popsize to standardised name * fix: benchmarks, coverage fix: benchmarks * fix: attribute selection in Optimisation class * adds changelog entry --- CHANGELOG.md | 1 + benchmarks/benchmark_optim_construction.py | 2 +- benchmarks/benchmark_parameterisation.py | 2 +- .../benchmark_track_parameterisation.py | 4 +- .../notebooks/optimiser_calibration.ipynb | 2 +- pybop/optimisers/base_pints_optimiser.py | 152 +++++---- pybop/optimisers/optimisation.py | 20 +- pybop/optimisers/pints_optimisers.py | 290 ++++++++++++++++-- pybop/optimisers/scipy_optimisers.py | 150 ++++++--- tests/unit/test_optimisation.py | 74 +++-- 10 files changed, 501 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eba964c2e..4f0458b2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ## Optimisations - [#512](https://github.com/pybop-team/PyBOP/pull/513) - Refactors `LogPosterior` with attributes pointing to composed likelihood object. +- [#551](https://github.com/pybop-team/PyBOP/pull/551) - Refactors Optimiser arguments, `population_size` and `max_iterations` as default args, improves optimiser docstrings ## Bug Fixes diff --git a/benchmarks/benchmark_optim_construction.py b/benchmarks/benchmark_optim_construction.py index 0849b4f38..ccf60a045 100644 --- a/benchmarks/benchmark_optim_construction.py +++ b/benchmarks/benchmark_optim_construction.py @@ -19,7 +19,7 @@ def setup(self, model, parameter_set, optimiser): Args: model (pybop.Model): The model class to be benchmarked. parameter_set (str): The name of the parameter set to be used. - optimiser (pybop.Optimiser): The optimizer class to be used. + optimiser (pybop.Optimiser): The optimiser class to be used. """ # Set random seed set_random_seed() diff --git a/benchmarks/benchmark_parameterisation.py b/benchmarks/benchmark_parameterisation.py index 9615f7876..4e723da8f 100644 --- a/benchmarks/benchmark_parameterisation.py +++ b/benchmarks/benchmark_parameterisation.py @@ -124,4 +124,4 @@ def time_optimiser_ask(self, model, parameter_set, optimiser): optimiser (pybop.Optimiser): The optimizer class being used. """ if optimiser not in [pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution]: - self.optim.pints_optimiser.ask() + self.optim.optimiser.ask() diff --git a/benchmarks/benchmark_track_parameterisation.py b/benchmarks/benchmark_track_parameterisation.py index be0fe5068..ccd8c3f88 100644 --- a/benchmarks/benchmark_track_parameterisation.py +++ b/benchmarks/benchmark_track_parameterisation.py @@ -29,7 +29,7 @@ def setup(self, model, parameter_set, optimiser): Args: model (pybop.Model): The model class to be benchmarked. parameter_set (str): The name of the parameter set to be used. - optimiser (pybop.Optimiser): The optimizer class to be used. + optimiser (pybop.Optimiser): The optimiser class to be used. """ # Set random seed set_random_seed() @@ -121,7 +121,7 @@ def results_tracking(self, model, parameter_set, optimiser): Args: model (pybop.Model): The model class being benchmarked (unused). parameter_set (str): The name of the parameter set being used (unused). - optimiser (pybop.Optimiser): The optimizer class being used (unused). + optimiser (pybop.Optimiser): The optimiser class being used (unused). """ results = self.optim.run() return results.x diff --git a/examples/notebooks/optimiser_calibration.ipynb b/examples/notebooks/optimiser_calibration.ipynb index 02913084e..53a34a063 100644 --- a/examples/notebooks/optimiser_calibration.ipynb +++ b/examples/notebooks/optimiser_calibration.ipynb @@ -610,7 +610,7 @@ "source": [ "for optim, sigma in zip(optims, sigmas):\n", " print(\n", - " f\"| Sigma: {sigma} | Num Iterations: {optim.result.n_iterations} | Best Cost: {optim.pints_optimiser.f_best()} | Results: {optim.pints_optimiser.x_best()} |\"\n", + " f\"| Sigma: {sigma} | Num Iterations: {optim.result.n_iterations} | Best Cost: {optim.optimiser.f_best()} | Results: {optim.optimiser.x_best()} |\"\n", " )" ] }, diff --git a/pybop/optimisers/base_pints_optimiser.py b/pybop/optimisers/base_pints_optimiser.py index a5d7c9abe..06a05409c 100644 --- a/pybop/optimisers/base_pints_optimiser.py +++ b/pybop/optimisers/base_pints_optimiser.py @@ -21,6 +21,18 @@ class BasePintsOptimiser(BaseOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + pints_optimiser : class + The PINTS optimiser class to be used. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like @@ -30,27 +42,51 @@ class BasePintsOptimiser(BaseOptimiser): bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and upper bounds on the parameters. + use_f_guessed : bool + Whether to track guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. """ - def __init__(self, cost, pints_optimiser, **optimiser_kwargs): + def __init__( + self, + cost, + pints_optimiser, + max_iterations: int = None, + min_iterations: int = 2, + max_unchanged_iterations: int = 15, + parallel: bool = False, + **optimiser_kwargs, + ): # First set attributes to default values self._boundaries = None self._needs_sensitivities = None self._use_f_guessed = None - self._parallel = False self._n_workers = 1 self._callback = None - self._max_iterations = None - self._min_iterations = 2 - self._unchanged_max_iterations = 15 + self.set_parallel(parallel) + self.set_max_iterations(max_iterations) + self.set_min_iterations(min_iterations) + self._unchanged_max_iterations = max_unchanged_iterations self._absolute_tolerance = 1e-5 self._relative_tolerance = 1e-2 self._max_evaluations = None self._threshold = None self._evaluations = None self._iterations = None + self.option_methods = { + "use_f_guessed": self.set_f_guessed_tracking, + "max_evaluations": self.set_max_evaluations, + "threshold": self.set_threshold, + } - self.pints_optimiser = pints_optimiser + self.optimiser = pints_optimiser super().__init__(cost, **optimiser_kwargs) def _set_up_optimiser(self): @@ -61,47 +97,26 @@ def _set_up_optimiser(self): self._sanitise_inputs() # Create an instance of the PINTS optimiser class - if issubclass(self.pints_optimiser, PintsOptimiser): - self.pints_optimiser = self.pints_optimiser( + if issubclass(self.optimiser, PintsOptimiser): + self.optimiser = self.optimiser( self.x0, sigma0=self.sigma0, boundaries=self._boundaries ) else: - raise ValueError( - "The pints_optimiser is not a recognised PINTS optimiser class." - ) + raise ValueError("The optimiser is not a recognised PINTS optimiser class.") # Check if sensitivities are required - self._needs_sensitivities = self.pints_optimiser.needs_sensitivities() - - # Apply default maxiter - self.set_max_iterations() + self._needs_sensitivities = self.optimiser.needs_sensitivities() # Apply additional options and remove them from options - key_list = list(self.unset_options.keys()) - for key in key_list: - if key == "use_f_guessed": - self.set_f_guessed_tracking(self.unset_options.pop(key)) - elif key == "parallel": - self.set_parallel(self.unset_options.pop(key)) - elif key == "max_iterations": - self.set_max_iterations(self.unset_options.pop(key)) - elif key == "min_iterations": - self.set_min_iterations(self.unset_options.pop(key)) - elif key == "max_unchanged_iterations": - max_unchanged_kwargs = {"iterations": self.unset_options.pop(key)} - if "absolute_tolerance" in self.unset_options.keys(): - max_unchanged_kwargs["absolute_tolerance"] = self.unset_options.pop( - "absolute_tolerance" - ) - if "relative_tolerance" in self.unset_options.keys(): - max_unchanged_kwargs["relative_tolerance"] = self.unset_options.pop( - "relative_tolerance" - ) - self.set_max_unchanged_iterations(**max_unchanged_kwargs) - elif key == "max_evaluations": - self.set_max_evaluations(self.unset_options.pop(key)) - elif key == "threshold": - self.set_threshold(self.unset_options.pop(key)) + max_unchanged_kwargs = {"iterations": self._unchanged_max_iterations} + for key, method in self.option_methods.items(): + if key in self.unset_options: + method(self.unset_options.pop(key)) + + # Capture tolerance options + for tol_key in ["absolute_tolerance", "relative_tolerance"]: + if tol_key in self.unset_options: + max_unchanged_kwargs[tol_key] = self.unset_options.pop(tol_key) def _sanitise_inputs(self): """ @@ -119,48 +134,29 @@ def _sanitise_inputs(self): ) self.unset_options.pop("options") - # Check for duplicate keywords - expected_keys = ["max_iterations", "popsize"] - alternative_keys = ["maxiter", "population_size"] - for exp_key, alt_key in zip(expected_keys, alternative_keys): - if alt_key in self.unset_options.keys(): - if exp_key in self.unset_options.keys(): - raise Exception( - "The alternative {alt_key} option was passed in addition to the expected {exp_key} option." - ) - else: # rename - self.unset_options[exp_key] = self.unset_options.pop(alt_key) - # Convert bounds to PINTS boundaries if self.bounds is not None: ignored_optimisers = (PintsGradientDescent, PintsAdam, PintsNelderMead) - if issubclass(self.pints_optimiser, ignored_optimisers): - print(f"NOTE: Boundaries ignored by {self.pints_optimiser}") + if issubclass(self.optimiser, ignored_optimisers): + print(f"NOTE: Boundaries ignored by {self.optimiser}") self.bounds = None else: - if issubclass(self.pints_optimiser, PintsPSO): + if issubclass(self.optimiser, PintsPSO): if not all( np.isfinite(value) for sublist in self.bounds.values() for value in sublist ): raise ValueError( - f"Either all bounds or no bounds must be set for {self.pints_optimiser.__name__}." + f"Either all bounds or no bounds must be set for {self.optimiser.__name__}." ) self._boundaries = PintsRectangularBoundaries( self.bounds["lower"], self.bounds["upper"] ) def name(self): - """ - Provides the name of the optimisation strategy. - - Returns - ------- - str - The name given by PINTS. - """ - return self.pints_optimiser.name() + """Returns the name of the PINTS optimisation strategy.""" + return self.optimiser.name() def _run(self): """ @@ -211,8 +207,8 @@ def fun(x): # For population based optimisers, don't use more workers than # particles! - if isinstance(self.pints_optimiser, PintsPopulationBasedOptimiser): - n_workers = min(n_workers, self.pints_optimiser.population_size()) + if isinstance(self.optimiser, PintsPopulationBasedOptimiser): + n_workers = min(n_workers, self.optimiser.population_size()) evaluator = PintsParallelEvaluator(fun, n_workers=n_workers) else: evaluator = PintsSequentialEvaluator(fun) @@ -231,17 +227,17 @@ def fun(x): try: while running: # Ask optimiser for new points - xs = self.pints_optimiser.ask() + xs = self.optimiser.ask() # Evaluate points fs = evaluator.evaluate(xs) # Tell optimiser about function values - self.pints_optimiser.tell(fs) + self.optimiser.tell(fs) # Update the scores - fb = self.pints_optimiser.f_best() - fg = self.pints_optimiser.f_guessed() + fb = self.optimiser.f_best() + fg = self.optimiser.f_guessed() fg_user = (fb, fg) if self.minimising else (-fb, -fg) # Check for significant changes against the absolute and relative tolerance @@ -260,7 +256,7 @@ def fun(x): _fs = [x[0] for x in fs] if self._needs_sensitivities else fs self.log_update( x=xs, - x_best=self.pints_optimiser.x_best(), + x_best=self.optimiser.x_best(), cost=_fs if self.minimising else [-x for x in _fs], cost_best=[fb] if self.minimising else [-fb], ) @@ -313,7 +309,7 @@ def fun(x): ) # Error in optimiser - error = self.pints_optimiser.stop() + error = self.optimiser.stop() if error: running = False halt_message = str(error) @@ -329,7 +325,7 @@ def fun(x): print("Current position:") # Show current parameters - x_user = self.pints_optimiser.x_guessed() + x_user = self.optimiser.x_guessed() if self._transformation: x_user = self._transformation.to_model(x_user) for p in x_user: @@ -347,11 +343,11 @@ def fun(x): # Get best parameters if self._use_f_guessed: - x = self.pints_optimiser.x_guessed() - f = self.pints_optimiser.f_guessed() + x = self.optimiser.x_guessed() + f = self.optimiser.f_guessed() else: - x = self.pints_optimiser.x_best() - f = self.pints_optimiser.f_best() + x = self.optimiser.x_best() + f = self.optimiser.f_best() # Inverse transform search parameters if self._transformation: diff --git a/pybop/optimisers/optimisation.py b/pybop/optimisers/optimisation.py index c2669163a..59d5c8e1a 100644 --- a/pybop/optimisers/optimisation.py +++ b/pybop/optimisers/optimisation.py @@ -32,24 +32,24 @@ class Optimisation: """ def __init__(self, cost, optimiser=None, **optimiser_kwargs): - self.__dict__["optimiser"] = ( + self.__dict__["optim"] = ( None # Pre-define optimiser to avoid recursion during initialisation ) if optimiser is None: - self.optimiser = XNES(cost, **optimiser_kwargs) + self.optim = XNES(cost, **optimiser_kwargs) elif issubclass(optimiser, BasePintsOptimiser): - self.optimiser = optimiser(cost, **optimiser_kwargs) + self.optim = optimiser(cost, **optimiser_kwargs) elif issubclass(optimiser, BaseSciPyOptimiser): - self.optimiser = optimiser(cost, **optimiser_kwargs) + self.optim = optimiser(cost, **optimiser_kwargs) else: raise ValueError("Unknown optimiser type") def run(self): - return self.optimiser.run() + return self.optim.run() def __getattr__(self, attr): - if "optimiser" in self.__dict__ and hasattr(self.optimiser, attr): - return getattr(self.optimiser, attr) + if "optim" in self.__dict__ and hasattr(self.optim, attr): + return getattr(self.optim, attr) raise AttributeError( f"'{self.__class__.__name__}' object has no attribute '{attr}'" ) @@ -57,9 +57,9 @@ def __getattr__(self, attr): def __setattr__(self, name: str, value) -> None: if ( name in self.__dict__ - or "optimiser" not in self.__dict__ - or not hasattr(self.optimiser, name) + or "optim" not in self.__dict__ + or not hasattr(self.optim, name) ): object.__setattr__(self, name, value) else: - setattr(self.optimiser, name, value) + setattr(self.optim, name, value) diff --git a/pybop/optimisers/pints_optimisers.py b/pybop/optimisers/pints_optimisers.py index 86dba3100..0637d6673 100644 --- a/pybop/optimisers/pints_optimisers.py +++ b/pybop/optimisers/pints_optimisers.py @@ -14,7 +14,7 @@ class GradientDescent(BasePintsOptimiser): """ - Implements a simple gradient descent optimization algorithm. + Implements a simple gradient descent optimisation algorithm. This class extends the gradient descent optimiser from the PINTS library, designed to minimize a scalar function of one or more variables. @@ -23,25 +23,64 @@ class GradientDescent(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like Initial position from which optimisation will start. sigma0 : float - The learning rate / Initial step size. + Initial step size or standard deviation depending on the optimiser. + bounds : dict + A dictionary with 'lower' and 'upper' keys containing arrays for lower and + upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- pints.GradientDescent : The PINTS implementation this class is based on. """ - def __init__(self, cost, **optimiser_kwargs): - super().__init__(cost, PintsGradientDescent, **optimiser_kwargs) + def __init__( + self, + cost, + max_iterations: int = None, + min_iterations: int = 2, + max_unchanged_iterations: int = 15, + parallel: bool = False, + **optimiser_kwargs, + ): + super().__init__( + cost, + PintsGradientDescent, + max_iterations, + min_iterations, + max_unchanged_iterations, + parallel, + **optimiser_kwargs, + ) class Adam(BasePintsOptimiser): """ - Implements the Adam optimization algorithm. + Implements the Adam optimisation algorithm. This class extends the Adam optimiser from the PINTS library, which combines ideas from RMSProp and Stochastic Gradient Descent with momentum. @@ -50,12 +89,35 @@ class Adam(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like Initial position from which optimisation will start. sigma0 : float - Initial step size. + Initial step size or standard deviation depending on the optimiser. + bounds : dict + A dictionary with 'lower' and 'upper' keys containing arrays for lower and + upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -82,14 +144,38 @@ class AdamW(BasePintsOptimiser): using larger learning rates. Note that this optimiser does not support boundary constraints. + Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional - Valid PyBOP option keys and their values, for example: + Valid PINTS option keys and their values, for example: x0 : array_like Initial position from which optimisation will start. sigma0 : float - Initial step size. + Initial step size or standard deviation depending on the optimiser. + bounds : dict + A dictionary with 'lower' and 'upper' keys containing arrays for lower and + upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -102,7 +188,7 @@ def __init__(self, cost, **optimiser_kwargs): class IRPropMin(BasePintsOptimiser): """ - Implements the iRpropMin optimization algorithm. + Implements the iRpropMin optimisation algorithm. This class inherits from the PINTS IRPropMin class, which is an optimiser that uses resilient backpropagation with weight-backtracking. It is designed to handle @@ -110,15 +196,35 @@ class IRPropMin(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like Initial position from which optimisation will start. sigma0 : float - Initial step size. + Initial step size or standard deviation depending on the optimiser. bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -131,23 +237,43 @@ def __init__(self, cost, **optimiser_kwargs): class PSO(BasePintsOptimiser): """ - Implements a particle swarm optimization (PSO) algorithm. + Implements a particle swarm optimisation (PSO) algorithm. This class extends the PSO optimiser from the PINTS library. PSO is a - metaheuristic optimization method inspired by the social behavior of birds - flocking or fish schooling, suitable for global optimization problems. + metaheuristic optimisation method inspired by the social behavior of birds + flocking or fish schooling, suitable for global optimisation problems. Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like - Initial positions of particles, which the optimisation will use. + Initial position from which optimisation will start. sigma0 : float - Spread of the initial particle positions. + Initial step size or standard deviation depending on the optimiser. bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -160,7 +286,7 @@ def __init__(self, cost, **optimiser_kwargs): class SNES(BasePintsOptimiser): """ - Implements the stochastic natural evolution strategy (SNES) optimization algorithm. + Implements the stochastic natural evolution strategy (SNES) optimisation algorithm. Inheriting from the PINTS SNES class, this optimiser is an evolutionary algorithm that evolves a probability distribution on the parameter space, guiding the search @@ -168,15 +294,35 @@ class SNES(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like Initial position from which optimisation will start. sigma0 : float - Initial standard deviation of the sampling distribution. + Initial step size or standard deviation depending on the optimiser. bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -197,15 +343,35 @@ class XNES(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like - The initial parameter vector to optimise. + Initial position from which optimisation will start. sigma0 : float - Initial standard deviation of the sampling distribution. + Initial step size or standard deviation depending on the optimiser. bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and - upperbounds on the parameters. If ``None``, no bounds are enforced. + upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -228,13 +394,35 @@ class NelderMead(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like - The initial parameter vector to optimise. + Initial position from which optimisation will start. sigma0 : float - Initial standard deviation of the sampling distribution. - Does not appear to be used. + Initial step size or standard deviation depending on the optimiser. + bounds : dict + A dictionary with 'lower' and 'upper' keys containing arrays for lower and + upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -249,21 +437,41 @@ class CMAES(BasePintsOptimiser): """ Adapter for the Covariance Matrix Adaptation Evolution Strategy (CMA-ES) optimiser in PINTS. - CMA-ES is an evolutionary algorithm for difficult non-linear non-convex optimization problems. + CMA-ES is an evolutionary algorithm for difficult non-linear non-convex optimisation problems. It adapts the covariance matrix of a multivariate normal distribution to capture the shape of the cost landscape. Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional Valid PINTS option keys and their values, for example: x0 : array_like - The initial parameter vector to optimise. + Initial position from which optimisation will start. sigma0 : float - Initial standard deviation of the sampling distribution. + Initial step size or standard deviation depending on the optimiser. bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and - upper bounds on the parameters. If ``None``, no bounds are enforced. + upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- @@ -289,19 +497,39 @@ class CuckooSearch(BasePintsOptimiser): Parameters ---------- + cost : callable + The cost function to be minimized. + max_iterations : int, optional + Maximum number of iterations for the optimisation. + min_iterations : int, optional (default=2) + Minimum number of iterations before termination. + max_unchanged_iterations : int, optional (default=15) + Maximum number of iterations without improvement before termination. + parallel : bool, optional (default=False) + Whether to run the optimisation in parallel. **optimiser_kwargs : optional - Valid PyBOP option keys and their values, for example: + Valid PINTS option keys and their values, for example: x0 : array_like - Initial parameter values. + Initial position from which optimisation will start. sigma0 : float - Initial step size. + Initial step size or standard deviation depending on the optimiser. bounds : dict A dictionary with 'lower' and 'upper' keys containing arrays for lower and upper bounds on the parameters. + use_f_guessed : bool + Whether to return the guessed function values. + absolute_tolerance : float + Absolute tolerance for convergence checking. + relative_tolerance : float + Relative tolerance for convergence checking. + max_evaluations : int + Maximum number of function evaluations. + threshold : float + Threshold value for early termination. See Also -------- - pybop.CuckooSearch : PyBOP implementation of Cuckoo Search algorithm. + pybop.CuckooSearchImpl : PyBOP implementation of Cuckoo Search algorithm. """ def __init__(self, cost, **optimiser_kwargs): diff --git a/pybop/optimisers/scipy_optimisers.py b/pybop/optimisers/scipy_optimisers.py index 52c6ba1fc..064cf3f02 100644 --- a/pybop/optimisers/scipy_optimisers.py +++ b/pybop/optimisers/scipy_optimisers.py @@ -23,8 +23,9 @@ class BaseSciPyOptimiser(BaseOptimiser): """ def __init__(self, cost, **optimiser_kwargs): - super().__init__(cost, **optimiser_kwargs) self.num_resamples = 40 + self.key_mapping = {"max_iterations": "maxiter", "population_size": "popsize"} + super().__init__(cost, **optimiser_kwargs) def _sanitise_inputs(self): """ @@ -42,17 +43,15 @@ def _sanitise_inputs(self): ) self.unset_options.pop("options") - # Check for duplicate keywords - expected_keys = ["maxiter", "popsize"] - alternative_keys = ["max_iterations", "population_size"] - for exp_key, alt_key in zip(expected_keys, alternative_keys): - if alt_key in self.unset_options.keys(): - if exp_key in self.unset_options.keys(): + # Convert PyBOP keys to SciPy + for pybop_key, scipy_key in self.key_mapping.items(): + if pybop_key in self.unset_options: + if scipy_key in self.unset_options: raise Exception( - "The alternative {alt_key} option was passed in addition to the expected {exp_key} option." + f"The alternative {pybop_key} option was passed in addition to the expected {scipy_key} option." ) - else: # rename - self.unset_options[exp_key] = self.unset_options.pop(alt_key) + # Rename the key + self.unset_options[scipy_key] = self.unset_options.pop(pybop_key) # Convert bounds to SciPy format if isinstance(self.bounds, dict): @@ -103,27 +102,58 @@ def _run(self): class SciPyMinimize(BaseSciPyOptimiser): """ - Adapts SciPy's minimize function for use as an optimization strategy. + Adapts SciPy's minimize function for use as an optimisation strategy. - This class provides an interface to various scalar minimization algorithms implemented in SciPy, - allowing fine-tuning of the optimization process through method selection and option configuration. + This class provides an interface to various scalar minimisation algorithms implemented in SciPy, + allowing fine-tuning of the optimisation process through method selection and option configuration. Parameters ---------- **optimiser_kwargs : optional - Valid SciPy Minimize option keys and their values, For example: + Valid SciPy Minimize option keys and their values: x0 : array_like Initial position from which optimisation will start. - bounds : dict, sequence or scipy.optimize.Bounds - Bounds for variables as supported by the selected method. method : str The optimisation method, options include: 'Nelder-Mead', 'Powell', 'CG', 'BFGS', 'Newton-CG', 'L-BFGS-B', 'TNC', 'COBYLA', 'SLSQP', 'trust-constr', 'dogleg', 'trust-ncg', 'trust-exact', 'trust-krylov'. + jac : {callable, '2-point', '3-point', 'cs', bool}, optional + Method for computing the gradient vector. + hess : {callable, '2-point', '3-point', 'cs', HessianUpdateStrategy}, optional + Method for computing the Hessian matrix. + hessp : callable, optional + Hessian of objective function times an arbitrary vector p. + bounds : sequence or scipy.optimize.Bounds, optional + Bounds on variables for L-BFGS-B, TNC, SLSQP, trust-constr methods. + constraints : {Constraint, dict} or List of {Constraint, dict}, optional + Constraints definition for constrained optimisation. + tol : float, optional + Tolerance for termination. + options : dict, optional + Method-specific options. Common options include: + maxiter : int + Maximum number of iterations. + disp : bool + Set to True to print convergence messages. + ftol : float + Function tolerance for termination. + gtol : float + Gradient tolerance for termination. + eps : float + Step size for finite difference approximation. + maxfev : int + Maximum number of function evaluations. + maxcor : int + Maximum number of variable metric corrections (L-BFGS-B). See Also -------- scipy.optimize.minimize : The SciPy method this class is based on. + + Notes + ----- + Different optimisation methods may support different options. Consult SciPy's + documentation for method-specific options and constraints. """ def __init__(self, cost, **optimiser_kwargs): @@ -251,40 +281,85 @@ def base_callback(intermediate_result: Union[OptimizeResult, np.ndarray]): ) def name(self): - """ - Provides the name of the optimization strategy. - - Returns - ------- - str - The name 'SciPyMinimize'. - """ + """Provides the name of the optimisation strategy.""" return "SciPyMinimize" class SciPyDifferentialEvolution(BaseSciPyOptimiser): """ - Adapts SciPy's differential_evolution function for global optimization. + Adapts SciPy's differential_evolution function for global optimisation. - This class provides a global optimization strategy based on differential evolution, useful for + This class provides a global optimisation strategy based on differential evolution, useful for problems involving continuous parameters and potentially multiple local minima. Parameters ---------- bounds : dict, sequence or scipy.optimize.Bounds Bounds for variables. Must be provided as it is essential for differential evolution. + Each element is a tuple (min, max) for the corresponding variable. **optimiser_kwargs : optional - Valid SciPy option keys and their values, for example: - strategy : str - The differential evolution strategy to use. - maxiter : int - Maximum number of iterations to perform. - popsize : int - The number of individuals in the population. + Valid SciPy differential_evolution options: + strategy : str, optional + The differential evolution strategy to use. Should be one of: + - 'best1bin' + - 'best1exp' + - 'rand1exp' + - 'randtobest1exp' + - 'currenttobest1exp' + - 'best2exp' + - 'rand2exp' + - 'randtobest1bin' + - 'currenttobest1bin' + - 'best2bin' + - 'rand2bin' + - 'rand1bin' + Default is 'best1bin'. + maxiter : int, optional + Maximum number of generations. Default is 1000. + popsize : int, optional + Multiplier for setting the total population size. The population has + popsize * len(x) individuals. Default is 15. + tol : float, optional + Relative tolerance for convergence. Default is 0.01. + mutation : float or tuple(float, float), optional + The mutation constant. If specified as a float, should be in [0, 2]. + If specified as a tuple (min, max), dithering is used. Default is (0.5, 1.0). + recombination : float, optional + The recombination constant, should be in [0, 1]. Default is 0.7. + seed : int, optional + Random seed for reproducibility. + disp : bool, optional + Display status messages. Default is False. + callback : callable, optional + Called after each iteration with the current result as argument. + polish : bool, optional + If True, performs a local optimisation on the solution. Default is True. + init : str or array-like, optional + Specify initial population. Can be 'latinhypercube', 'random', + or an array of shape (M, len(x)). + atol : float, optional + Absolute tolerance for convergence. Default is 0. + updating : {'immediate', 'deferred'}, optional + If 'immediate', best solution vector is continuously updated within + a single generation. Default is 'immediate'. + workers : int or map-like callable, optional + If workers is an int the population is subdivided into workers + sections and evaluated in parallel. Default is 1. + constraints : {NonlinearConstraint, LinearConstraint, Bounds}, optional + Constraints on the solver. See Also -------- scipy.optimize.differential_evolution : The SciPy method this class is based on. + + Notes + ----- + Differential Evolution is a stochastic population based method that is useful for + global optimisation problems. At each pass through the population the algorithm mutates + each candidate solution by mixing with other candidate solutions to create a trial + candidate. The fitness of all candidates is then evaluated and for each candidate if + the trial candidate is an improvement, it takes its place in the population for the next + iteration. """ def __init__(self, cost, **optimiser_kwargs): @@ -374,12 +449,5 @@ def cost_wrapper(x): ) def name(self): - """ - Provides the name of the optimization strategy. - - Returns - ------- - str - The name 'SciPyDifferentialEvolution'. - """ + """Provides the name of the optimisation strategy.""" return "SciPyDifferentialEvolution" diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index 0b746980b..10422ef43 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -141,7 +141,7 @@ def test_no_optimisation_parameters(self, model, dataset): ) @pytest.mark.unit def test_optimiser_kwargs(self, cost, optimiser): - optim = optimiser(cost=cost, maxiter=3, tol=1e-6) + optim = optimiser(cost=cost, max_iterations=3, tol=1e-6) cost_bounds = cost.parameters.get_bounds() # Check maximum iterations @@ -198,7 +198,7 @@ def test_optimiser_kwargs(self, cost, optimiser): ): warnings.simplefilter("always") optim = optimiser(cost=cost, unrecognised=10) - assert not optim.pints_optimiser.running() + assert not optim.optimiser.running() else: # Check bounds in list format and update tol bounds = [ @@ -213,23 +213,28 @@ def test_optimiser_kwargs(self, cost, optimiser): pybop.XNES, ]: # Pass nested options - optim = optimiser(cost=cost, options=dict(maxiter=10)) + optim = optimiser(cost=cost, options=dict(max_iterations=10)) with pytest.raises( Exception, - match="A duplicate maxiter option was found in the options dictionary.", + match="A duplicate max_evaluations option was found in the options dictionary.", ): - optimiser(cost=cost, maxiter=5, options=dict(maxiter=10)) + optimiser( + cost=cost, max_evaluations=5, options=dict(max_evaluations=10) + ) - # Pass similar keywords - with pytest.raises( - Exception, - match="option was passed in addition to the expected", - ): - optimiser(cost=cost, maxiter=5, max_iterations=10) + if optimiser in [pybop.SciPyDifferentialEvolution, pybop.SciPyMinimize]: + # Pass duplicate keywords + with pytest.raises( + Exception, + match="option was passed in addition to the expected", + ): + optimiser(cost=cost, maxiter=5, max_iterations=10) if optimiser in [pybop.SciPyDifferentialEvolution]: - # Update population size - optimiser(cost=cost, popsize=5) + # Update population size, test maxiter arg + pop_maxiter_optim = optimiser(cost=cost, maxiter=3, popsize=5) + assert pop_maxiter_optim._options["maxiter"] == 3 + assert pop_maxiter_optim._options["popsize"] == 5 # Test invalid bounds with pytest.raises( @@ -248,34 +253,34 @@ def test_optimiser_kwargs(self, cost, optimiser): # Test AdamW hyperparameters if optimiser in [pybop.AdamW]: optim = optimiser(cost=cost, b1=0.9, b2=0.999, lam=0.1) - optim.pints_optimiser.b1 = 0.9 - optim.pints_optimiser.b2 = 0.9 - optim.pints_optimiser.lam = 0.1 + optim.optimiser.b1 = 0.9 + optim.optimiser.b2 = 0.9 + optim.optimiser.lam = 0.1 - assert optim.pints_optimiser.b1 == 0.9 - assert optim.pints_optimiser.b2 == 0.9 - assert optim.pints_optimiser.lam == 0.1 + assert optim.optimiser.b1 == 0.9 + assert optim.optimiser.b2 == 0.9 + assert optim.optimiser.lam == 0.1 # Incorrect values for i, _match in (("Value", -1),): with pytest.raises( Exception, match="must be a numeric value between 0 and 1." ): - optim.pints_optimiser.b1 = i + optim.optimiser.b1 = i with pytest.raises( Exception, match="must be a numeric value between 0 and 1." ): - optim.pints_optimiser.b2 = i + optim.optimiser.b2 = i with pytest.raises( Exception, match="must be a numeric value between 0 and 1." ): - optim.pints_optimiser.lam = i + optim.optimiser.lam = i # Check defaults - assert optim.pints_optimiser.n_hyper_parameters() == 5 - assert optim.pints_optimiser.x_guessed() == optim.pints_optimiser._x0 + assert optim.optimiser.n_hyper_parameters() == 5 + assert optim.optimiser.x_guessed() == optim.optimiser._x0 with pytest.raises(RuntimeError): - optim.pints_optimiser.tell([0.1]) + optim.optimiser.tell([0.1]) else: # Check and update initial values @@ -290,12 +295,14 @@ def test_optimiser_kwargs(self, cost, optimiser): def test_cuckoo_no_bounds(self, cost): optim = pybop.CuckooSearch(cost=cost, bounds=None, max_iterations=1) optim.run() - assert optim.pints_optimiser._boundaries is None + assert optim.optimiser._boundaries is None @pytest.mark.unit def test_scipy_minimize_with_jac(self, cost): # Check a method that uses gradient information - optim = pybop.SciPyMinimize(cost=cost, method="L-BFGS-B", jac=True, maxiter=1) + optim = pybop.SciPyMinimize( + cost=cost, method="L-BFGS-B", jac=True, max_iterations=1 + ) results = optim.run() assert results.get_scipy_result() == optim.result.scipy_result @@ -310,7 +317,10 @@ def test_scipy_minimize_invalid_x0(self, cost): # Check a starting point that returns an infinite cost invalid_x0 = np.array([1.1]) optim = pybop.SciPyMinimize( - cost=cost, x0=invalid_x0, maxiter=10, allow_infeasible_solutions=False + cost=cost, + x0=invalid_x0, + max_iterations=10, + allow_infeasible_solutions=False, ) optim.run() assert abs(optim._cost0) != np.inf @@ -363,7 +373,7 @@ class RandomClass: with pytest.raises( ValueError, - match="The pints_optimiser is not a recognised PINTS optimiser class.", + match="The optimiser is not a recognised PINTS optimiser class.", ): pybop.BasePintsOptimiser(cost=cost, pints_optimiser=RandomClass) @@ -545,7 +555,7 @@ def callback_error(iteration, s): def optimiser_error(): return "Optimiser error message" - optim.pints_optimiser.stop = optimiser_error + optim.optimiser.stop = optimiser_error optim.run() assert optim.result.n_iterations == 1 @@ -563,7 +573,9 @@ def optimiser_error(): def test_infeasible_solutions(self, cost): # Test infeasible solutions for optimiser in [pybop.SciPyMinimize, pybop.GradientDescent]: - optim = optimiser(cost=cost, allow_infeasible_solutions=False, maxiter=1) + optim = optimiser( + cost=cost, allow_infeasible_solutions=False, max_iterations=1 + ) optim.run() assert optim.result.n_iterations == 1