From 79a3ce9a6c529774d266922fd0b447bf57a8d039 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Sat, 27 Jul 2019 16:35:14 +0800 Subject: [PATCH 1/5] change quniform style in GridSearch to conform with other tuners --- docs/en_US/Tutorial/SearchSpaceSpec.md | 6 ----- .../nni/gridsearch_tuner/gridsearch_tuner.py | 22 ++++++++----------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index f5a1d34d4f..b4ca2e81ef 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -90,12 +90,6 @@ All types of sampling strategies and their parameter are listed here: Known Limitations: -* Note that In Grid Search Tuner, for users' convenience, the definition of `quniform` and `qloguniform` change, where q here specifies the number of values that will be sampled. Details about them are listed as follows - - * Type 'quniform' will receive three values [low, high, q], where [low, high] specifies a range and 'q' specifies the number of values that will be sampled evenly. Note that q should be at least 2. It will be sampled in a way that the first sampled value is 'low', and each of the following values is (high-low)/q larger that the value in front of it. - - * Type 'qloguniform' behaves like 'quniform' except that it will first change the range to [log(low), log(high)] and sample and then change the sampled value back. - * Note that Metis Tuner only supports numerical `choice` now * Note that for nested search space: diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index 98c43d1cc6..4aa778a65c 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -43,9 +43,8 @@ class GridSearchTuner(Tuner): Type 'choice' will select one of the options. Note that it can also be nested. - Type 'quniform' will receive three values [low, high, q], where [low, high] specifies a range and 'q' specifies the number of values that will be sampled evenly. - Note that q should be at least 2. - It will be sampled in a way that the first sampled value is 'low', and each of the following values is (high-low)/q larger that the value in front of it. + Type 'quniform' will receive three values [low, high, q], where [low, high] specifies a range and 'q' specifies the interval + It will be sampled in a way that the first sampled value is 'low', and each of the following values is 'interval' larger that the value in front of it. Type 'qloguniform' behaves like 'quniform' except that it will first change the range to [log(low), log(high)] and sample and then change the sampled value back. @@ -95,19 +94,16 @@ def json2parameter(self, ss_spec): def _parse_quniform(self, param_value): '''parse type of quniform parameter and return a list''' - if param_value[2] < 2: - raise RuntimeError("The number of values sampled (q) should be at least 2") - low, high, count = param_value[0], param_value[1], param_value[2] - interval = (high - low) / (count - 1) - return [float(low + interval * i) for i in range(count)] + low, high, interval = param_value[0], param_value[1], param_value[2] + count = int(np.floor((high - low) / interval)) + 1 + return [low + interval * i for i in range(count)] def parse_qtype(self, param_type, param_value): '''parse type of quniform or qloguniform''' - if param_type == 'quniform': - return self._parse_quniform(param_value) - if param_type == 'qloguniform': - param_value[:2] = np.log(param_value[:2]) - return list(np.exp(self._parse_quniform(param_value))) + if param_type in ['quniform', 'qloguniform']: + # qloguniform has no difference from quniform, + # since grid search does not consider possibility distribution + return self._parse_quniform(param_value) raise RuntimeError("Not supported type: %s" % param_type) From 4117f1bc93573cb402205e2f4de4f268ca67b4eb Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Thu, 1 Aug 2019 10:24:35 +0800 Subject: [PATCH 2/5] update doc --- docs/en_US/Tuner/BuiltinTuner.md | 4 ++-- src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en_US/Tuner/BuiltinTuner.md b/docs/en_US/Tuner/BuiltinTuner.md index 2b07978850..cfd34f8637 100644 --- a/docs/en_US/Tuner/BuiltinTuner.md +++ b/docs/en_US/Tuner/BuiltinTuner.md @@ -14,7 +14,7 @@ Currently we support the following algorithms: |[__Naïve Evolution__](#Evolution)|Naïve Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naïve Evolution requires many trials to works, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)| |[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC need to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)| |[__Batch tuner__](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.| -|[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, qloguniform. The number q in quniform and qloguniform has special meaning (different from the spec in search space spec). It means the number of values that will be sampled evenly from the range low and high.| +|[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, qloguniform. | |[__Hyperband__](#Hyperband)|Hyperband tries to use the limited resource to explore as many configurations as possible, and finds out the promising ones to get the final result. The basic idea is generating many configurations and to run them for the small number of trial budget to find out promising one, then further training those promising ones to select several more promising one.[Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)| |[__Network Morphism__](#NetworkMorphism)|Network Morphism provides functions to automatically search for architecture of deep learning models. Every child network inherits the knowledge from its parent network and morphs into diverse types of networks, including changes of depth, width, and skip-connection. Next, it estimates the value of a child network using the historic architecture and metric pairs. Then it selects the most promising one to train. [Reference Paper](https://arxiv.org/abs/1806.10282)| |[__Metis Tuner__](#MetisTuner)|Metis offers the following benefits when it comes to tuning parameters: While most tools only predict the optimal configuration, Metis gives you two outputs: (a) current prediction of optimal configuration, and (b) suggestion for the next trial. No more guesswork. While most tools assume training datasets do not have noisy data, Metis actually tells you if you need to re-sample a particular hyper-parameter. [Reference Paper](https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/)| @@ -211,7 +211,7 @@ The search space file including the high-level key `combine_params`. The type of **Suggested scenario** -Note that the only acceptable types of search space are `choice`, `quniform`, `qloguniform`. **The number `q` in `quniform` and `qloguniform` has special meaning (different from the spec in [search space spec](../Tutorial/SearchSpaceSpec.md)). It means the number of values that will be sampled evenly from the range `low` and `high`.** +Note that the only acceptable types of search space are `choice`, `quniform`, `qloguniform`. It is suggested when search space is small, it is feasible to exhaustively sweeping the whole search space. [Detailed Description](./GridsearchTuner.md) diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index 4aa778a65c..6db278303e 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -44,7 +44,7 @@ class GridSearchTuner(Tuner): Type 'choice' will select one of the options. Note that it can also be nested. Type 'quniform' will receive three values [low, high, q], where [low, high] specifies a range and 'q' specifies the interval - It will be sampled in a way that the first sampled value is 'low', and each of the following values is 'interval' larger that the value in front of it. + It will be sampled in a way that the first sampled value is 'low', and each of the following values is 'interval' larger than the value in front of it. Type 'qloguniform' behaves like 'quniform' except that it will first change the range to [log(low), log(high)] and sample and then change the sampled value back. From e4424985f20d69f8c183f43506fe7ffa1c465163 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Thu, 1 Aug 2019 14:06:37 +0800 Subject: [PATCH 3/5] Grid Search: delete qloguniform; add randint --- docs/en_US/Tuner/BuiltinTuner.md | 4 ++-- docs/en_US/Tutorial/SearchSpaceSpec.md | 2 +- .../nni/gridsearch_tuner/gridsearch_tuner.py | 24 +++++++++---------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/en_US/Tuner/BuiltinTuner.md b/docs/en_US/Tuner/BuiltinTuner.md index cfd34f8637..086ed2819d 100644 --- a/docs/en_US/Tuner/BuiltinTuner.md +++ b/docs/en_US/Tuner/BuiltinTuner.md @@ -14,7 +14,7 @@ Currently we support the following algorithms: |[__Naïve Evolution__](#Evolution)|Naïve Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naïve Evolution requires many trials to works, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)| |[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC need to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)| |[__Batch tuner__](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.| -|[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, qloguniform. | +|[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, randint. | |[__Hyperband__](#Hyperband)|Hyperband tries to use the limited resource to explore as many configurations as possible, and finds out the promising ones to get the final result. The basic idea is generating many configurations and to run them for the small number of trial budget to find out promising one, then further training those promising ones to select several more promising one.[Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)| |[__Network Morphism__](#NetworkMorphism)|Network Morphism provides functions to automatically search for architecture of deep learning models. Every child network inherits the knowledge from its parent network and morphs into diverse types of networks, including changes of depth, width, and skip-connection. Next, it estimates the value of a child network using the historic architecture and metric pairs. Then it selects the most promising one to train. [Reference Paper](https://arxiv.org/abs/1806.10282)| |[__Metis Tuner__](#MetisTuner)|Metis offers the following benefits when it comes to tuning parameters: While most tools only predict the optimal configuration, Metis gives you two outputs: (a) current prediction of optimal configuration, and (b) suggestion for the next trial. No more guesswork. While most tools assume training datasets do not have noisy data, Metis actually tells you if you need to re-sample a particular hyper-parameter. [Reference Paper](https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/)| @@ -211,7 +211,7 @@ The search space file including the high-level key `combine_params`. The type of **Suggested scenario** -Note that the only acceptable types of search space are `choice`, `quniform`, `qloguniform`. +Note that the only acceptable types of search space are `choice`, `quniform`, `randint`. It is suggested when search space is small, it is feasible to exhaustively sweeping the whole search space. [Detailed Description](./GridsearchTuner.md) diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index b4ca2e81ef..4354ebaae4 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -82,7 +82,7 @@ All types of sampling strategies and their parameter are listed here: | Evolution Tuner | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | SMAC Tuner | ✓ | ✓ | ✓ | ✓ | ✓ | | | | | | | Batch Tuner | ✓ | | | | | | | | | | -| Grid Search Tuner | ✓ | | | ✓ | | ✓ | | | | | +| Grid Search Tuner | ✓ | ✓ | | ✓ | | | | | | | | Hyperband Advisor | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | Metis Tuner | ✓ | ✓ | ✓ | ✓ | | | | | | | | GP Tuner | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | | | | diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index 6db278303e..0a2d7d9d69 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -39,15 +39,14 @@ class GridSearchTuner class GridSearchTuner(Tuner): ''' GridSearchTuner will search all the possible configures that the user define in the searchSpace. - The only acceptable types of search space are 'quniform', 'qloguniform' and 'choice' + The only acceptable types of search space are 'choice', 'quniform', 'randint' Type 'choice' will select one of the options. Note that it can also be nested. Type 'quniform' will receive three values [low, high, q], where [low, high] specifies a range and 'q' specifies the interval It will be sampled in a way that the first sampled value is 'low', and each of the following values is 'interval' larger than the value in front of it. - Type 'qloguniform' behaves like 'quniform' except that it will first change the range to [log(low), log(high)] - and sample and then change the sampled value back. + Type 'randint' gives all possible intergers in range(low, high). Note that 'high' is not included. ''' def __init__(self): @@ -72,8 +71,12 @@ def json2parameter(self, ss_spec): chosen_params.extend(choice) else: chosen_params.append(choice) + elif _type == 'quniform': + chosen_params = self._parse_quniform(_value) + elif _type == 'randint': + chosen_params = self._parse_randint(_value) else: - chosen_params = self.parse_qtype(_type, _value) + raise RuntimeError("Not supported type: %s" % _type) else: chosen_params = dict() for key in ss_spec.keys(): @@ -98,14 +101,9 @@ def _parse_quniform(self, param_value): count = int(np.floor((high - low) / interval)) + 1 return [low + interval * i for i in range(count)] - def parse_qtype(self, param_type, param_value): - '''parse type of quniform or qloguniform''' - if param_type in ['quniform', 'qloguniform']: - # qloguniform has no difference from quniform, - # since grid search does not consider possibility distribution - return self._parse_quniform(param_value) - - raise RuntimeError("Not supported type: %s" % param_type) + def _parse_randint(self, param_value): + '''parse type of randint parameter and return a list''' + return np.arange(param_value[0], param_value[1]).tolist() def expand_parameters(self, para): ''' @@ -129,7 +127,7 @@ def expand_parameters(self, para): def update_search_space(self, search_space): ''' - Check if the search space is valid and expand it: only contains 'choice' type or other types beginnning with the letter 'q' + Check if the search space is valid and expand it: only contains 'choice', 'randint', '' ''' self.expanded_search_space = self.json2parameter(search_space) From d248881057076d13abb308fc730746731536933e Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Thu, 1 Aug 2019 14:10:21 +0800 Subject: [PATCH 4/5] fix typo --- src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index 0a2d7d9d69..cd6985c79b 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -127,7 +127,7 @@ def expand_parameters(self, para): def update_search_space(self, search_space): ''' - Check if the search space is valid and expand it: only contains 'choice', 'randint', '' + Check if the search space is valid and expand it: support only 'choice', 'quniform', randint' ''' self.expanded_search_space = self.json2parameter(search_space) From e3dc24a3933817ed76bc77d5166ec07cf5668613 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Fri, 2 Aug 2019 00:38:26 +0800 Subject: [PATCH 5/5] update comments --- src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index cd6985c79b..f6af325c56 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -46,7 +46,7 @@ class GridSearchTuner(Tuner): Type 'quniform' will receive three values [low, high, q], where [low, high] specifies a range and 'q' specifies the interval It will be sampled in a way that the first sampled value is 'low', and each of the following values is 'interval' larger than the value in front of it. - Type 'randint' gives all possible intergers in range(low, high). Note that 'high' is not included. + Type 'randint' gives all possible intergers in range[low, high). Note that 'high' is not included. ''' def __init__(self):