From 835efa13d38f6ed0db1be7ca5f5f58730896f496 Mon Sep 17 00:00:00 2001 From: Junhwa Song Date: Mon, 19 Dec 2022 13:25:02 +0900 Subject: [PATCH] [Feature] Support HyperBand and BOHB scheduler (#101) * Bump ray from 1.9.1 to 2.1.0 Signed-off-by: Junhwa Song --- README.md | 9 +++--- configs/_base_/scheduler/bohb.py | 2 ++ configs/_base_/scheduler/hb.py | 2 ++ configs/_base_/searcher/bohb.py | 1 + .../_base_/searcher/nevergrad_oneplusone.py | 5 +-- configs/_base_/searcher/nevergrad_pso.py | 6 +--- requirements/optional.txt | 2 ++ siatune/ray/schedulers/__init__.py | 4 +-- siatune/ray/schedulers/builder.py | 9 +++--- siatune/ray/schedulers/pbt.py | 4 +-- siatune/ray/searchers/__init__.py | 7 +---- siatune/ray/searchers/builder.py | 4 +++ siatune/ray/searchers/flaml.py | 15 --------- siatune/ray/searchers/hyperopt.py | 9 ------ siatune/ray/searchers/nevergrad.py | 6 ++-- tests/test_ray/test_schedulers.py | 4 +-- tests/test_ray/test_searchers.py | 31 ++++++++++++++----- 17 files changed, 57 insertions(+), 63 deletions(-) create mode 100644 configs/_base_/scheduler/bohb.py create mode 100644 configs/_base_/scheduler/hb.py create mode 100644 configs/_base_/searcher/bohb.py delete mode 100644 siatune/ray/searchers/flaml.py delete mode 100644 siatune/ray/searchers/hyperopt.py diff --git a/README.md b/README.md index b3f31681..860616cd 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,12 @@ SIATune is an open-source deep learning model hyperparameter tuning toolbox espe - **Schedule multiple experiments** Various scheduling techniques are supported to efficiently manage many experiments. - - [x] [AsyncHyperBandScheduler](https://arxiv.org/abs/1810.05934) - - [ ] [PopulationBasedTraining](https://www.deepmind.com/blog/population-based-training-of-neural-networks) - - [ ] [MedianStoppingRule](https://research.google.com/pubs/pub46180.html) + - [x] [Asynchronous HyperBand](https://arxiv.org/abs/1810.05934) + - [x] [HyperBand](https://arxiv.org/abs/1603.06560) + - [ ] [Median Stopping Rule](https://research.google.com/pubs/pub46180.html) + - [ ] [Population Based Training](https://www.deepmind.com/blog/population-based-training-of-neural-networks) - [ ] [Population Based Bandits](https://arxiv.org/abs/2002.02518) - - [ ] [HyperBandScheduler](https://arxiv.org/abs/1603.06560) + - [x] [Bayesian Optimization and HyperBand](https://arxiv.org/abs/1807.01774) - **Distributed tuning system based on Ray** diff --git a/configs/_base_/scheduler/bohb.py b/configs/_base_/scheduler/bohb.py new file mode 100644 index 00000000..ad530a80 --- /dev/null +++ b/configs/_base_/scheduler/bohb.py @@ -0,0 +1,2 @@ +trial_scheduler = dict( + type='HyperBandForBOHB', time_attr='training_iteration', max_t=20) diff --git a/configs/_base_/scheduler/hb.py b/configs/_base_/scheduler/hb.py new file mode 100644 index 00000000..a291300e --- /dev/null +++ b/configs/_base_/scheduler/hb.py @@ -0,0 +1,2 @@ +trial_scheduler = dict( + type='HyperBandScheduler', time_attr='training_iteration', max_t=20) diff --git a/configs/_base_/searcher/bohb.py b/configs/_base_/searcher/bohb.py new file mode 100644 index 00000000..52aac5b2 --- /dev/null +++ b/configs/_base_/searcher/bohb.py @@ -0,0 +1 @@ +searcher = dict(type='TuneBOHB') diff --git a/configs/_base_/searcher/nevergrad_oneplusone.py b/configs/_base_/searcher/nevergrad_oneplusone.py index dfb991b5..a0025c11 100644 --- a/configs/_base_/searcher/nevergrad_oneplusone.py +++ b/configs/_base_/searcher/nevergrad_oneplusone.py @@ -1,4 +1 @@ -searcher = dict( - type='NevergradSearch', - budget=256, -) +searcher = dict(type='NevergradSearch', budget=256) diff --git a/configs/_base_/searcher/nevergrad_pso.py b/configs/_base_/searcher/nevergrad_pso.py index b3a2ad6b..6ea9bd22 100644 --- a/configs/_base_/searcher/nevergrad_pso.py +++ b/configs/_base_/searcher/nevergrad_pso.py @@ -1,5 +1 @@ -searcher = dict( - type='NevergradSearch', - optimizer='PSO', - budget=256, -) +searcher = dict(type='NevergradSearch', optimizer='PSO', budget=256) diff --git a/requirements/optional.txt b/requirements/optional.txt index 4ce60537..46c1be81 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -1,5 +1,7 @@ bayesian-optimization==1.2.0 +ConfigSpace flaml==1.0.14 +hpbandster hyperopt==0.2.5 mlflow==1.23.1 nevergrad==0.4.3.post7 diff --git a/siatune/ray/schedulers/__init__.py b/siatune/ray/schedulers/__init__.py index 44cfa967..94e0dcad 100644 --- a/siatune/ray/schedulers/__init__.py +++ b/siatune/ray/schedulers/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) SI-Analytics. All rights reserved. -from .builder import SCHEDULERS, build_scheduler +from .builder import TRIAL_SCHEDULERS, build_scheduler from .pbt import PopulationBasedTraining -__all__ = ['SCHEDULERS', 'build_scheduler', 'PopulationBasedTraining'] +__all__ = ['TRIAL_SCHEDULERS', 'build_scheduler', 'PopulationBasedTraining'] diff --git a/siatune/ray/schedulers/builder.py b/siatune/ray/schedulers/builder.py index f25cf185..720cc7da 100644 --- a/siatune/ray/schedulers/builder.py +++ b/siatune/ray/schedulers/builder.py @@ -3,15 +3,16 @@ from mmcv.utils import Config, Registry from ray import tune +from ray.tune.schedulers import TrialScheduler -SCHEDULERS = Registry('schedulers') +TRIAL_SCHEDULERS = Registry('trial scheduler') for v in set(tune.schedulers.SCHEDULER_IMPORT.values()): if not inspect.isclass(v): continue - SCHEDULERS.register_module(module=v) + TRIAL_SCHEDULERS.register_module(module=v) -def build_scheduler(cfg: Config) -> tune.schedulers.TrialScheduler: +def build_scheduler(cfg: Config) -> TrialScheduler: """Build the scheduler from configs. Args: @@ -20,4 +21,4 @@ def build_scheduler(cfg: Config) -> tune.schedulers.TrialScheduler: tune.schedulers.TrialScheduler: The scheduler. """ - return SCHEDULERS.build(cfg) + return TRIAL_SCHEDULERS.build(cfg) diff --git a/siatune/ray/schedulers/pbt.py b/siatune/ray/schedulers/pbt.py index 9280e9bc..73b4dcaf 100644 --- a/siatune/ray/schedulers/pbt.py +++ b/siatune/ray/schedulers/pbt.py @@ -8,7 +8,7 @@ PopulationBasedTraining as _PopulationBasedTraining from ray.tune.search.sample import Domain -from siatune.ray.schedulers import SCHEDULERS +from siatune.ray.schedulers import TRIAL_SCHEDULERS from siatune.ray.spaces import build_space from siatune.utils import ImmutableContainer @@ -50,7 +50,7 @@ def explore( return new_config -@SCHEDULERS.register_module(force=True) +@TRIAL_SCHEDULERS.register_module(force=True) class PopulationBasedTraining(_PopulationBasedTraining): def __init__(self, *args, **kwargs) -> None: diff --git a/siatune/ray/searchers/__init__.py b/siatune/ray/searchers/__init__.py index 0acc6537..96760fac 100644 --- a/siatune/ray/searchers/__init__.py +++ b/siatune/ray/searchers/__init__.py @@ -1,10 +1,5 @@ # Copyright (c) SI-Analytics. All rights reserved. from .builder import SEARCHERS, build_searcher -from .flaml import BlendSearch, CFOSearch -from .hyperopt import HyperOptSearch from .nevergrad import NevergradSearch -__all__ = [ - 'SEARCHERS', 'build_searcher', 'BlendSearch', 'CFOSearch', - 'HyperOptSearch', 'NevergradSearch' -] +__all__ = ['SEARCHERS', 'build_searcher', 'NevergradSearch'] diff --git a/siatune/ray/searchers/builder.py b/siatune/ray/searchers/builder.py index 2d81a137..da8f6717 100644 --- a/siatune/ray/searchers/builder.py +++ b/siatune/ray/searchers/builder.py @@ -1,8 +1,12 @@ # Copyright (c) SI-Analytics. All rights reserved. + from mmcv.utils import Config, Registry +from ray import tune from ray.tune.search import Searcher SEARCHERS = Registry('searchers') +for func in set(tune.search.SEARCH_ALG_IMPORT.values()): + SEARCHERS.register_module(module=func()) def build_searcher(cfg: Config) -> Searcher: diff --git a/siatune/ray/searchers/flaml.py b/siatune/ray/searchers/flaml.py deleted file mode 100644 index b984c1e8..00000000 --- a/siatune/ray/searchers/flaml.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) SI-Analytics. All rights reserved. -from ray.tune.search.flaml import CFO as _CFO -from ray.tune.search.flaml import BlendSearch as _BlendSearch - -from .builder import SEARCHERS - - -@SEARCHERS.register_module() -class BlendSearch(_BlendSearch): - __doc__ = _BlendSearch.__doc__ - - -@SEARCHERS.register_module() -class CFOSearch(_CFO): - __doc__ = _CFO.__doc__ diff --git a/siatune/ray/searchers/hyperopt.py b/siatune/ray/searchers/hyperopt.py deleted file mode 100644 index 37921cfc..00000000 --- a/siatune/ray/searchers/hyperopt.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) SI-Analytics. All rights reserved. -from ray.tune.search.hyperopt import HyperOptSearch as _HyperOptSearch - -from .builder import SEARCHERS - - -@SEARCHERS.register_module() -class HyperOptSearch(_HyperOptSearch): - __doc__ = _HyperOptSearch.__doc__ diff --git a/siatune/ray/searchers/nevergrad.py b/siatune/ray/searchers/nevergrad.py index d4620f8b..317f3f8e 100644 --- a/siatune/ray/searchers/nevergrad.py +++ b/siatune/ray/searchers/nevergrad.py @@ -21,7 +21,7 @@ optimizer_registry = dict() -@SEARCHERS.register_module() +@SEARCHERS.register_module(force=True) class NevergradSearch(_NevergradSearch): """Search with Nevergrad.""" @@ -104,8 +104,8 @@ def _setup_nevergrad(self) -> None: if len(self._nevergrad_opt.instrumentation.args) != 1: raise ValueError( 'Instrumented optimizers should use kwargs only') - if self._parameters is not None and \ - self._nevergrad_opt.dimension != len(self._parameters): + if self._parameters is not None and (self._nevergrad_opt.dimension != + len(self._parameters)): raise ValueError('len(parameters_names) must match optimizer ' 'dimension for non-instrumented optimizers') diff --git a/tests/test_ray/test_schedulers.py b/tests/test_ray/test_schedulers.py index 244c0ea0..82a24915 100644 --- a/tests/test_ray/test_schedulers.py +++ b/tests/test_ray/test_schedulers.py @@ -1,9 +1,9 @@ -from siatune.ray.schedulers import SCHEDULERS, build_scheduler +from siatune.ray.schedulers import TRIAL_SCHEDULERS, build_scheduler def test_build_schedulers(): - @SCHEDULERS.register_module() + @TRIAL_SCHEDULERS.register_module() class TestScheduler: pass diff --git a/tests/test_ray/test_searchers.py b/tests/test_ray/test_searchers.py index 640eed47..2000af70 100644 --- a/tests/test_ray/test_searchers.py +++ b/tests/test_ray/test_searchers.py @@ -1,9 +1,7 @@ import pytest from ray import tune -from siatune.ray.searchers import (SEARCHERS, BlendSearch, CFOSearch, - HyperOptSearch, NevergradSearch, - build_searcher) +from siatune.ray.searchers import SEARCHERS, build_searcher def test_build_searcher(): @@ -39,7 +37,17 @@ def test_blend(trainable, config): trainable, metric='mean_loss', mode='min', - search_alg=BlendSearch(), + search_alg=build_searcher(dict(type='BlendSearch')), + num_samples=2, + config=config) + + +def test_bohb(trainable, config): + tune.run( + trainable, + metric='mean_loss', + mode='min', + search_alg=build_searcher(dict(type='TuneBOHB')), num_samples=2, config=config) @@ -49,7 +57,7 @@ def test_cfo(trainable, config): trainable, metric='mean_loss', mode='min', - search_alg=CFOSearch(), + search_alg=build_searcher(dict(type='CFO')), num_samples=2, config=config) @@ -59,7 +67,7 @@ def test_hyperopt(trainable, config): trainable, metric='mean_loss', mode='min', - search_alg=HyperOptSearch(), + search_alg=build_searcher(dict(type='HyperOptSearch')), num_samples=2, config=config) @@ -69,6 +77,15 @@ def test_nevergrad(trainable, config): trainable, metric='mean_loss', mode='min', - search_alg=NevergradSearch(optimizer='PSO', budget=2), + search_alg=build_searcher(dict(type='NevergradSearch', budget=1)), + num_samples=2, + config=config) + + tune.run( + trainable, + metric='mean_loss', + mode='min', + search_alg=build_searcher( + dict(type='NevergradSearch', optimizer='PSO', budget=1)), num_samples=2, config=config)