From bac8a01a42321936e3837898ed7c7735b54f6496 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 10:31:24 +0800 Subject: [PATCH 01/26] update docstring of batchtuner --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 32 ++++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index 4342c89ab1..1b2a367f74 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -31,7 +31,7 @@ class BatchTuner CHOICE = 'choice' VALUE = '_value' -logger = logging.getLogger('batch_tuner_AutoML') +LOGGER = logging.getLogger('batch_tuner_AutoML') class BatchTuner(Tuner): """ @@ -42,6 +42,13 @@ class BatchTuner(Tuner): '_value': '[{...}, {...}, {...}]', } } + + Attributes + ---------- + count : int + count is the number of parameters that tuner has generated. + values : list + values is to save all the candidates contains in search space. """ def __init__(self): @@ -62,11 +69,14 @@ def is_valid(self, search_space): for param in search_space: param_type = search_space[param][TYPE] if not param_type == CHOICE: - raise RuntimeError('BatchTuner only supprt one combined-paramreters type is choice.') - else: - if isinstance(search_space[param][VALUE], list): - return search_space[param][VALUE] - raise RuntimeError('The combined-paramreters value in BatchTuner is not a list.') + raise RuntimeError('BatchTuner only supprt \ + one combined-paramreters type is choice.') + + if isinstance(search_space[param][VALUE], list): + return search_space[param][VALUE] + + raise RuntimeError('The combined-paramreters \ + value in BatchTuner is not a list.') return None def update_search_space(self, search_space): @@ -101,7 +111,7 @@ def import_data(self, data): a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' """ if len(self.values) == 0: - logger.info("Search space has not been initialized, skip this data import") + LOGGER .info("Search space has not been initialized, skip this data import") return self.values = self.values[(self.count+1):] @@ -109,16 +119,18 @@ def import_data(self, data): _completed_num = 0 for trial_info in data: - logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + LOGGER .info("Importing data, current processing \ + progress %s / %s", _completed_num, len(data)) # simply validate data format assert "parameter" in trial_info _params = trial_info["parameter"] assert "value" in trial_info _value = trial_info['value'] if not _value: - logger.info("Useless trial data, value is %s, skip this trial data.", _value) + LOGGER .info("Useless trial data, value is %s, skip this trial data.", _value) continue _completed_num += 1 if _params in self.values: self.values.remove(_params) - logger.info("Successfully import data to batch tuner, total data: %d, imported data: %d.", len(data), _completed_num) + LOGGER .info("Successfully import data to batch tuner, \ + total data: %d, imported data: %d.", len(data), _completed_num) From d10da159be56f6d09b4f1b1af63805256a26a9bf Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 11:42:02 +0800 Subject: [PATCH 02/26] update docstring of batch tuner --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index 1b2a367f74..2237f251df 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -62,6 +62,11 @@ def is_valid(self, search_space): Parameters ---------- search_space : dict + + Returns + ------- + None or list + If valid, return candidate values; else return None. """ if not len(search_space) == 1: raise RuntimeError('BatchTuner only supprt one combined-paramreters key.') @@ -94,6 +99,11 @@ def generate_parameters(self, parameter_id, **kwargs): Parameters ---------- parameter_id : int + + Returns + ------- + dict + A candidate parameter group. """ self.count += 1 if self.count > len(self.values) - 1: @@ -105,6 +115,7 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): def import_data(self, data): """Import additional data for tuning + Parameters ---------- data: From 6ab848122cfe420d7844d0a2867280802405fdd2 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 12:00:43 +0800 Subject: [PATCH 03/26] update docstring of evolution tuner --- .../nni/evolution_tuner/evolution_tuner.py | 78 ++++++++++++++----- 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py index edad05663b..f5d5187b5f 100644 --- a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py +++ b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py @@ -32,7 +32,9 @@ def json2space(x, oldy=None, name=NodeType.ROOT): - """Change search space from json format to hyperopt format + """ + Change search space from json format to hyperopt format + """ y = list() if isinstance(x, dict): @@ -40,7 +42,7 @@ def json2space(x, oldy=None, name=NodeType.ROOT): _type = x[NodeType.TYPE] name = name + '-' + _type if _type == 'choice': - if oldy != None: + if oldy is not None: _index = oldy[NodeType.INDEX] y += json2space(x[NodeType.VALUE][_index], oldy[NodeType.VALUE], name=name+'[%d]' % _index) @@ -49,19 +51,20 @@ def json2space(x, oldy=None, name=NodeType.ROOT): y.append(name) else: for key in x.keys(): - y += json2space(x[key], (oldy[key] if oldy != - None else None), name+"[%s]" % str(key)) + y += json2space(x[key], \ + (oldy[key] if oldy is not None else None), name+"[%s]" % str(key)) elif isinstance(x, list): for i, x_i in enumerate(x): if isinstance(x_i, dict): if NodeType.NAME not in x_i.keys(): raise RuntimeError('\'_name\' key is not found in this nested search space.') - y += json2space(x_i, (oldy[i] if oldy != - None else None), name+"[%d]" % i) + y += json2space(x_i, (oldy[i] if oldy is not None else None), name+"[%d]" % i) return y def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT): - """Json to pramaters. + """ + Json to pramaters. + """ if isinstance(x, dict): if NodeType.TYPE in x.keys(): @@ -75,11 +78,11 @@ def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeTyp y = { NodeType.INDEX: _index, NodeType.VALUE: json2parameter(x[NodeType.VALUE][_index], - is_rand, - random_state, - None, - Rand, - name=name+"[%d]" % _index) + is_rand, + random_state, + None, + Rand, + name=name+"[%d]" % _index) } else: y = eval('parameter_expressions.' + @@ -89,8 +92,8 @@ def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeTyp else: y = dict() for key in x.keys(): - y[key] = json2parameter(x[key], is_rand, random_state, oldy[key] - if oldy != None else None, Rand, name + "[%s]" % str(key)) + y[key] = json2parameter(x[key], is_rand, random_state, oldy[key] \ + if oldy is not None else None, Rand, name + "[%s]" % str(key)) elif isinstance(x, list): y = list() for i, x_i in enumerate(x): @@ -98,14 +101,26 @@ def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeTyp if NodeType.NAME not in x_i.keys(): raise RuntimeError('\'_name\' key is not found in this nested search space.') y.append(json2parameter(x_i, is_rand, random_state, oldy[i] - if oldy != None else None, Rand, name + "[%d]" % i)) + if oldy is not None else None, Rand, name + "[%d]" % i)) else: y = copy.deepcopy(x) return y -class Individual(object): + +class Individual: """ Indicidual class to store the indv info. + + Attributes + ---------- + config : str + Search space. + info : str + The str to save information of individual. + result : float + The final metric of a individual. + store_dir : str + save_dir : str """ def __init__(self, config=None, info=None, result=None, save_dir=None): @@ -113,6 +128,7 @@ def __init__(self, config=None, info=None, result=None, save_dir=None): Parameters ---------- config : str + A config to represent a group of parameters. info : str result : float save_dir : str @@ -129,6 +145,8 @@ def __str__(self): def mutation(self, config=None, info=None, save_dir=None): """ + Mutation by reset state information. + Parameters ---------- config : str @@ -166,8 +184,11 @@ def __init__(self, optimize_mode, population_size=32): self.population = None self.space = None + def update_search_space(self, search_space): - """Update search space. + """ + Update search space. + Search_space contains the information that user pre-defined. Parameters @@ -180,15 +201,19 @@ def update_search_space(self, search_space): self.random_state = np.random.RandomState() self.population = [] is_rand = dict() + for item in self.space: is_rand[item] = True + for _ in range(self.population_size): config = json2parameter( self.searchspace_json, is_rand, self.random_state) self.population.append(Individual(config=config)) + def generate_parameters(self, parameter_id, **kwargs): - """Returns a dict of trial (hyper-)parameters, as a serializable object. + """ + This function will returns a dict of trial (hyper-)parameters, as a serializable object. Parameters ---------- @@ -197,14 +222,18 @@ def generate_parameters(self, parameter_id, **kwargs): Returns ------- config : dict + A group of candaidte parameters that evolution tuner generated. """ if not self.population: raise RuntimeError('The population is empty') + pos = -1 + for i in range(len(self.population)): if self.population[i].result is None: pos = i break + if pos != -1: indiv = copy.deepcopy(self.population[pos]) self.population.pop(pos) @@ -219,6 +248,7 @@ def generate_parameters(self, parameter_id, **kwargs): self.population[0].config) is_rand = dict() mutation_pos = space[random.randint(0, len(space)-1)] + for i in range(len(self.space)): is_rand[self.space[i]] = (self.space[i] == mutation_pos) config = json2parameter( @@ -227,21 +257,27 @@ def generate_parameters(self, parameter_id, **kwargs): # remove "_index" from config and save params-id total_config = config + self.total_data[parameter_id] = total_config config = split_index(total_config) + return config + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): - '''Record the result from a trial + """ + Record the result from a trial Parameters ---------- - parameters: dict + parameter_id : int + parameters : dict value : dict/float if value is dict, it should have "default" key. value is final metrics of the trial. - ''' + """ reward = extract_scalar_reward(value) + if parameter_id not in self.total_data: raise RuntimeError('Received parameter_id not in total_data.') # restore the paramsters contains "_index" From fb29435209b3a1f6d3b456c09587f727d899b6ba Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 13:35:27 +0800 Subject: [PATCH 04/26] update docstring and pylint of metis_tuner --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 399 +++++++++++++------ 1 file changed, 270 insertions(+), 129 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index 9cfc9710f9..4395ad4a48 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -16,18 +16,24 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +""" +metis_tuner.py +""" import copy import logging -import numpy as np + +from enum import Enum, unique +from multiprocessing.dummy import Pool as ThreadPool +import warnings import os import random import statistics import sys -import warnings -from enum import Enum, unique -from multiprocessing.dummy import Pool as ThreadPool + +import numpy as np import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation import nni.metis_tuner.lib_data as lib_data @@ -54,10 +60,45 @@ class MetisTuner(Tuner): More algorithm information you could reference here: https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/ + + Attributes + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability: float + The probability of Metis to select parameter from exploration instead of exploitation. """ - def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=False, - selection_num_starting_points=600, cold_start_num=10, exploration_probability=0.9): + def __init__( + self, + optimize_mode="maximize", + no_resampling=True, + no_candidates=False, + selection_num_starting_points=600, + cold_start_num=10, + exploration_probability=0.9): """ Parameters ---------- @@ -65,23 +106,34 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=F optimize_mode is a string that including two mode "maximize" and "minimize" no_resampling : bool - True or False. Should Metis consider re-sampling as part of the search strategy? - If you are confident that the training dataset is noise-free, then you do not need re-sampling. - - no_candidates: bool - True or False. Should Metis suggest parameters for the next benchmark? - If you do not plan to do more benchmarks, Metis can skip this step. - - selection_num_starting_points: int - how many times Metis should try to find the global optimal in the search space? - The higher the number, the longer it takes to output the solution. - - cold_start_num: int - Metis need some trial result to get cold start. when the number of trial result is less than - cold_start_num, Metis will randomly sample hyper-parameter for trial. - - exploration_probability: float + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability : float The probability of Metis to select parameter from exploration instead of exploitation. + + x_bounds : list + The constration of parameters. + + x_types : list + The type of parameters. """ self.samples_x = [] @@ -99,10 +151,12 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=F self.minimize_constraints_fun = None self.minimize_starting_points = None self.supplement_data_num = 0 - + self.x_bounds = None + self.x_types = None def update_search_space(self, search_space): - """Update the self.x_bounds and self.x_types by the search_space.json + """ + Update the self.x_bounds and self.x_types by the search_space.json Parameters ---------- @@ -121,12 +175,20 @@ def update_search_space(self, search_space): key_range = search_space[key]['_value'] idx = self.key_order.index(key) if key_type == 'quniform': - if key_range[2] == 1 and key_range[0].is_integer() and key_range[1].is_integer(): - self.x_bounds[idx] = [key_range[0], key_range[1]+1] + if key_range[2] == 1 and key_range[0].is_integer( + ) and key_range[1].is_integer(): + self.x_bounds[idx] = [key_range[0], key_range[1] + 1] self.x_types[idx] = 'range_int' else: low, high, q = key_range - bounds = np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high) + bounds = np.clip( + np.arange( + np.round( + low / q), + np.round( + high / q) + 1) * q, + low, + high) self.x_bounds[idx] = bounds self.x_types[idx] = 'discrete_int' elif key_type == 'randint': @@ -140,22 +202,27 @@ def update_search_space(self, search_space): for key_value in key_range: if not isinstance(key_value, (int, float)): - raise RuntimeError("Metis Tuner only support numerical choice.") + raise RuntimeError( + "Metis Tuner only support numerical choice.") self.x_types[idx] = 'discrete_int' else: - logger.info("Metis Tuner doesn't support this kind of variable: " + str(key_type)) - raise RuntimeError("Metis Tuner doesn't support this kind of variable: " + str(key_type)) + logger.info( + "Metis Tuner doesn't support this kind of variable: " + + str(key_type)) + raise RuntimeError( + "Metis Tuner doesn't support this kind of variable: " + + str(key_type)) else: logger.info("The format of search space is not a dict.") raise RuntimeError("The format of search space is not a dict.") - self.minimize_starting_points = _rand_init(self.x_bounds, self.x_types, \ - self.selection_num_starting_points) - + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) def _pack_output(self, init_parameter): - """Pack the output + """ + Pack the output Parameters ---------- @@ -170,12 +237,14 @@ def _pack_output(self, init_parameter): output[self.key_order[i]] = param return output - def generate_parameters(self, parameter_id, **kwargs): - """Generate next parameter for trial + """ + Generate next parameter for trial + If the number of trial result is lower than cold start number, metis will first random generate some parameters. - Otherwise, metis will choose the parameters by the Gussian Process Model and the Gussian Mixture Model. + Otherwise, metis will choose the parameters by + the Gussian Process Model and the Gussian Mixture Model. Parameters ---------- @@ -189,21 +258,26 @@ def generate_parameters(self, parameter_id, **kwargs): init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] results = self._pack_output(init_parameter) else: - self.minimize_starting_points = _rand_init(self.x_bounds, self.x_types, \ - self.selection_num_starting_points) - results = self._selection(self.samples_x, self.samples_y_aggregation, self.samples_y, - self.x_bounds, self.x_types, - threshold_samplessize_resampling=(None if self.no_resampling is True else 50), - no_candidates=self.no_candidates, - minimize_starting_points=self.minimize_starting_points, - minimize_constraints_fun=self.minimize_constraints_fun) + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + results = self._selection( + self.samples_x, + self.samples_y_aggregation, + self.samples_y, + self.x_bounds, + self.x_types, + threshold_samplessize_resampling=( + None if self.no_resampling is True else 50), + no_candidates=self.no_candidates, + minimize_starting_points=self.minimize_starting_points, + minimize_constraints_fun=self.minimize_constraints_fun) logger.info("Generate paramageters:\n" + str(results)) return results - def receive_trial_result(self, parameter_id, parameters, value, **kwargs): - """Tuner receive result from trial. + """ + Tuner receive result from trial. Parameters ---------- @@ -244,12 +318,19 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): # calculate y aggregation self.samples_y_aggregation.append([value]) - - def _selection(self, samples_x, samples_y_aggregation, samples_y, - x_bounds, x_types, max_resampling_per_x=3, - threshold_samplessize_exploitation=12, - threshold_samplessize_resampling=50, no_candidates=False, - minimize_starting_points=None, minimize_constraints_fun=None): + def _selection( + self, + samples_x, + samples_y_aggregation, + samples_y, + x_bounds, + x_types, + max_resampling_per_x=3, + threshold_samplessize_exploitation=12, + threshold_samplessize_resampling=50, + no_candidates=False, + minimize_starting_points=None, + minimize_constraints_fun=None): with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -260,7 +341,8 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, samples_size_unique = len(samples_y) # ===== STEP 1: Compute the current optimum ===== - gp_model = gp_create_model.create_model(samples_x, samples_y_aggregation) + gp_model = gp_create_model.create_model( + samples_x, samples_y_aggregation) lm_current = gp_selection.selection( "lm", samples_y_aggregation, @@ -272,12 +354,12 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, if not lm_current: return None logger.info({'hyperparameter': lm_current['hyperparameter'], - 'expected_mu': lm_current['expected_mu'], - 'expected_sigma': lm_current['expected_sigma'], - 'reason': "exploitation_gp"}) + 'expected_mu': lm_current['expected_mu'], + 'expected_sigma': lm_current['expected_sigma'], + 'reason': "exploitation_gp"}) if no_candidates is False: - # ===== STEP 2: Get recommended configurations for exploration ===== + # ===== STEP 2: Get recommended configurations for exploration ==== results_exploration = gp_selection.selection( "lc", samples_y_aggregation, @@ -288,11 +370,15 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, minimize_constraints_fun=minimize_constraints_fun) if results_exploration is not None: - if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: - temp_candidate = {'hyperparameter': results_exploration['hyperparameter'], - 'expected_mu': results_exploration['expected_mu'], - 'expected_sigma': results_exploration['expected_sigma'], - 'reason': "exploration"} + if _num_past_samples( + results_exploration['hyperparameter'], + samples_x, + samples_y) == 0: + temp_candidate = { + 'hyperparameter': results_exploration['hyperparameter'], + 'expected_mu': results_exploration['expected_mu'], + 'expected_sigma': results_exploration['expected_sigma'], + 'reason': "exploration"} candidates.append(temp_candidate) logger.info("DEBUG: 1 exploration candidate selected\n") @@ -300,70 +386,91 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, else: logger.info("DEBUG: No suitable exploration candidates were") - # ===== STEP 3: Get recommended configurations for exploitation ===== + # ===== STEP 3: Get recommended configurations for exploitation === if samples_size_all >= threshold_samplessize_exploitation: logger.info("Getting candidates for exploitation...\n") try: - gmm = gmm_create_model.create_model(samples_x, samples_y_aggregation) + gmm = gmm_create_model.create_model( + samples_x, samples_y_aggregation) if ("discrete_int" in x_types) or ("range_int" in x_types): - results_exploitation = gmm_selection.selection(x_bounds, x_types, - gmm['clusteringmodel_good'], - gmm['clusteringmodel_bad'], - minimize_starting_points, - minimize_constraints_fun=minimize_constraints_fun) + results_exploitation = gmm_selection.selection( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) else: - # If all parameters are of "range_continuous", let's use GMM to generate random starting points - results_exploitation = gmm_selection.selection_r(x_bounds, x_types, - gmm['clusteringmodel_good'], - gmm['clusteringmodel_bad'], - num_starting_points=self.selection_num_starting_points, - minimize_constraints_fun=minimize_constraints_fun) + # If all parameters are of "range_continuous", + # let's use GMM to generate random starting points + results_exploitation = gmm_selection.selection_r( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + num_starting_points=self.selection_num_starting_points, + minimize_constraints_fun=minimize_constraints_fun) if results_exploitation is not None: - if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: - temp_expected_mu, temp_expected_sigma = gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model']) - temp_candidate = {'hyperparameter': results_exploitation['hyperparameter'], - 'expected_mu': temp_expected_mu, - 'expected_sigma': temp_expected_sigma, - 'reason': "exploitation_gmm"} + if _num_past_samples( + results_exploitation['hyperparameter'], + samples_x, + samples_y) == 0: + temp_expected_mu, temp_expected_sigma = gp_prediction.predict( + results_exploitation['hyperparameter'], gp_model['model']) + temp_candidate = { + 'hyperparameter': results_exploitation['hyperparameter'], + 'expected_mu': temp_expected_mu, + 'expected_sigma': temp_expected_sigma, + 'reason': "exploitation_gmm"} candidates.append(temp_candidate) - logger.info("DEBUG: 1 exploitation_gmm candidate selected\n") + logger.info( + "DEBUG: 1 exploitation_gmm candidate selected\n") logger.info(temp_candidate) else: - logger.info("DEBUG: No suitable exploitation_gmm candidates were found\n") + logger.info( + "DEBUG: No suitable exploitation_gmm candidates were found\n") except ValueError as exception: # The exception: ValueError: Fitting the mixture model failed # because some components have ill-defined empirical covariance # (for instance caused by singleton or collapsed samples). - # Try to decrease the number of components, or increase reg_covar. - logger.info("DEBUG: No suitable exploitation_gmm candidates were found due to exception.") + # Try to decrease the number of components, or increase + # reg_covar. + logger.info( + "DEBUG: No suitable exploitation_gmm candidates were found due to exception.") logger.info(exception) # ===== STEP 4: Get a list of outliers ===== if (threshold_samplessize_resampling is not None) and \ - (samples_size_unique >= threshold_samplessize_resampling): + (samples_size_unique >= threshold_samplessize_resampling): logger.info("Getting candidates for re-sampling...\n") - results_outliers = gp_outlier_detection.outlierDetection_threaded(samples_x, samples_y_aggregation) + results_outliers = gp_outlier_detection.outlierDetection_threaded( + samples_x, samples_y_aggregation) if results_outliers is not None: for results_outlier in results_outliers: - if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: - temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\ - 'expected_mu': results_outlier['expected_mu'],\ - 'expected_sigma': results_outlier['expected_sigma'],\ - 'reason': "resampling"} + if _num_past_samples( + samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: + temp_candidate = { + 'hyperparameter': samples_x[ + results_outlier['samples_idx']], + 'expected_mu': results_outlier['expected_mu'], + 'expected_sigma': results_outlier['expected_sigma'], + 'reason': "resampling"} candidates.append(temp_candidate) logger.info("DEBUG: %d re-sampling candidates selected\n") logger.info(temp_candidate) else: - logger.info("DEBUG: No suitable resampling candidates were found\n") + logger.info( + "DEBUG: No suitable resampling candidates were found\n") if candidates: - # ===== STEP 5: Compute the information gain of each candidate towards the optimum ===== - logger.info("Evaluating information gain of %d candidates...\n") + # ===== STEP 5: Compute the information gain of each candidate + logger.info( + "Evaluating information gain of %d candidates...\n") next_improvement = 0 threads_inputs = [[ @@ -371,37 +478,48 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, minimize_constraints_fun, minimize_starting_points ] for candidate in candidates] threads_pool = ThreadPool(4) - # Evaluate what would happen if we actually sample each candidate - threads_results = threads_pool.map(_calculate_lowest_mu_threaded, threads_inputs) + # Evaluate what would happen if we actually sample each + # candidate + threads_results = threads_pool.map( + _calculate_lowest_mu_threaded, threads_inputs) threads_pool.close() threads_pool.join() for threads_result in threads_results: if threads_result['expected_lowest_mu'] < lm_current['expected_mu']: # Information gain - temp_improvement = threads_result['expected_lowest_mu'] - lm_current['expected_mu'] + temp_improvement = threads_result['expected_lowest_mu'] - \ + lm_current['expected_mu'] if next_improvement > temp_improvement: next_improvement = temp_improvement next_candidate = threads_result['candidate'] else: - # ===== STEP 6: If we have no candidates, randomly pick one ===== + # ===== STEP 6: If we have no candidates, randomly pick one === logger.info( "DEBUG: No candidates from exploration, exploitation,\ and resampling. We will random a candidate for next_candidate\n" ) - next_candidate = _rand_with_constraints(x_bounds, x_types) \ - if minimize_starting_points is None else minimize_starting_points[0] - next_candidate = lib_data.match_val_type(next_candidate, x_bounds, x_types) - expected_mu, expected_sigma = gp_prediction.predict(next_candidate, gp_model['model']) - next_candidate = {'hyperparameter': next_candidate, 'reason': "random", - 'expected_mu': expected_mu, 'expected_sigma': expected_sigma} - - # ===== STEP 7: If current optimal hyperparameter occurs in the history or exploration probability is less than the threshold, take next config as exploration step ===== + next_candidate = _rand_with_constraints( + x_bounds, + x_types) if minimize_starting_points is None else minimize_starting_points[0] + next_candidate = lib_data.match_val_type( + next_candidate, x_bounds, x_types) + expected_mu, expected_sigma = gp_prediction.predict( + next_candidate, gp_model['model']) + next_candidate = { + 'hyperparameter': next_candidate, + 'reason': "random", + 'expected_mu': expected_mu, + 'expected_sigma': expected_sigma} + + # STEP 7: If current optimal hyperparameter occurs in the history + # or exploration probability is less than the threshold, take next + # config as exploration step outputs = self._pack_output(lm_current['hyperparameter']) ap = random.uniform(0, 1) - if outputs in self.total_data or ap<=self.exploration_probability: + if outputs in self.total_data or ap <= self.exploration_probability: if next_candidate is not None: outputs = self._pack_output(next_candidate['hyperparameter']) else: @@ -411,36 +529,50 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, return outputs def import_data(self, data): - """Import additional data for tuning + """ + Import additional data for tuning + Parameters ---------- - data: - a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + data : a list of dict + each of which has at least two keys: 'parameter' and 'value'. """ _completed_num = 0 for trial_info in data: - logger.info("Importing data, current processing progress %s / %s" %(_completed_num, len(data))) + logger.info( + "Importing data, current processing progress %s / %s" % + (_completed_num, len(data))) _completed_num += 1 assert "parameter" in trial_info _params = trial_info["parameter"] assert "value" in trial_info _value = trial_info['value'] if not _value: - logger.info("Useless trial data, value is %s, skip this trial data." %_value) + logger.info( + "Useless trial data, value is %s, skip this trial data." % + _value) continue self.supplement_data_num += 1 - _parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)]) + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) self.total_data.append(_params) - self.receive_trial_result(parameter_id=_parameter_id, parameters=_params, value=_value) + self.receive_trial_result( + parameter_id=_parameter_id, + parameters=_params, + value=_value) logger.info("Successfully import data to metis tuner.") + def _rand_with_constraints(x_bounds, x_types): outputs = None x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] - x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints,\ - x_types_withconstraints, CONSTRAINT_LOWERBOUND, CONSTRAINT_UPPERBOUND) + x_val_withconstraints = lib_constraint_summation.rand( + x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) if not x_val_withconstraints: outputs = [None] * len(x_bounds) @@ -454,12 +586,18 @@ def _rand_with_constraints(x_bounds, x_types): def _calculate_lowest_mu_threaded(inputs): - [candidate, samples_x, samples_y, x_bounds, x_types, minimize_constraints_fun, minimize_starting_points] = inputs + [candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points] = inputs outputs = {"candidate": candidate, "expected_lowest_mu": None} - for expected_mu in [candidate['expected_mu'] + 1.96 * candidate['expected_sigma'], - candidate['expected_mu'] - 1.96 * candidate['expected_sigma']]: + for expected_mu in [ + candidate['expected_mu'] + + 1.96 * + candidate['expected_sigma'], + candidate['expected_mu'] - + 1.96 * + candidate['expected_sigma']]: temp_samples_x = copy.deepcopy(samples_x) temp_samples_y = copy.deepcopy(samples_y) @@ -472,8 +610,10 @@ def _calculate_lowest_mu_threaded(inputs): temp_samples_y.append([expected_mu]) # Aggregates multiple observation of the sample sampling points - temp_y_aggregation = [statistics.median(temp_sample_y) for temp_sample_y in temp_samples_y] - temp_gp = gp_create_model.create_model(temp_samples_x, temp_y_aggregation) + temp_y_aggregation = [statistics.median( + temp_sample_y) for temp_sample_y in temp_samples_y] + temp_gp = gp_create_model.create_model( + temp_samples_x, temp_y_aggregation) temp_results = gp_selection.selection( "lm", temp_y_aggregation, @@ -502,18 +642,19 @@ def _rand_init(x_bounds, x_types, selection_num_starting_points): ''' Random sample some init seed within bounds. ''' - return [lib_data.rand(x_bounds, x_types) for i \ - in range(0, selection_num_starting_points)] + return [lib_data.rand(x_bounds, x_types) for i + in range(0, selection_num_starting_points)] def get_median(temp_list): - """Return median + """ + Return median """ num = len(temp_list) temp_list.sort() print(temp_list) if num % 2 == 0: - median = (temp_list[int(num/2)] + temp_list[int(num/2) - 1]) / 2 + median = (temp_list[int(num / 2)] + temp_list[int(num / 2) - 1]) / 2 else: - median = temp_list[int(num/2)] + median = temp_list[int(num / 2)] return median From c6d9e95e42112cbe2a7e4c2947efd09ea736c493 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 13:43:40 +0800 Subject: [PATCH 05/26] fix pylint related to logger in metis_tuner --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 31 ++++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index 4395ad4a48..eec66956be 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -25,13 +25,10 @@ import copy import logging -from enum import Enum, unique from multiprocessing.dummy import Pool as ThreadPool import warnings -import os import random import statistics -import sys import numpy as np @@ -128,10 +125,10 @@ def __init__( exploration_probability : float The probability of Metis to select parameter from exploration instead of exploitation. - + x_bounds : list The constration of parameters. - + x_types : list The type of parameters. """ @@ -208,10 +205,10 @@ def update_search_space(self, search_space): self.x_types[idx] = 'discrete_int' else: logger.info( - "Metis Tuner doesn't support this kind of variable: " + + "Metis Tuner doesn't support this kind of variable: %s", str(key_type)) raise RuntimeError( - "Metis Tuner doesn't support this kind of variable: " + + "Metis Tuner doesn't support this kind of variable: %s", str(key_type)) else: logger.info("The format of search space is not a dict.") @@ -272,7 +269,7 @@ def generate_parameters(self, parameter_id, **kwargs): minimize_starting_points=self.minimize_starting_points, minimize_constraints_fun=self.minimize_constraints_fun) - logger.info("Generate paramageters:\n" + str(results)) + logger.info("Generate paramageters: \n%s", str(results)) return results def receive_trial_result(self, parameter_id, parameters, value, **kwargs): @@ -291,8 +288,8 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): value = -value logger.info("Received trial result.") - logger.info("value is :" + str(value)) - logger.info("parameter is : " + str(parameters)) + logger.info("value is : %s", str(value)) + logger.info("parameter is : %s", str(parameters)) # parse parameter to sample_x sample_x = [0 for i in range(len(self.key_order))] @@ -440,7 +437,8 @@ def _selection( # Try to decrease the number of components, or increase # reg_covar. logger.info( - "DEBUG: No suitable exploitation_gmm candidates were found due to exception.") + "DEBUG: No suitable exploitation_gmm \ + candidates were found due to exception.") logger.info(exception) # ===== STEP 4: Get a list of outliers ===== @@ -453,7 +451,8 @@ def _selection( if results_outliers is not None: for results_outlier in results_outliers: if _num_past_samples( - samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: + samples_x[results_outlier['samples_idx']], + samples_x, samples_y) < max_resampling_per_x: temp_candidate = { 'hyperparameter': samples_x[ results_outlier['samples_idx']], @@ -540,8 +539,7 @@ def import_data(self, data): _completed_num = 0 for trial_info in data: logger.info( - "Importing data, current processing progress %s / %s" % - (_completed_num, len(data))) + "Importing data, current processing progress %s / %s", _completed_num, len(data)) _completed_num += 1 assert "parameter" in trial_info _params = trial_info["parameter"] @@ -549,7 +547,7 @@ def import_data(self, data): _value = trial_info['value'] if not _value: logger.info( - "Useless trial data, value is %s, skip this trial data." % + "Useless trial data, value is %s, skip this trial data.", _value) continue self.supplement_data_num += 1 @@ -623,7 +621,8 @@ def _calculate_lowest_mu_threaded(inputs): minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) - if outputs["expected_lowest_mu"] is None or outputs["expected_lowest_mu"] > temp_results['expected_mu']: + if outputs["expected_lowest_mu"] is None \ + or outputs["expected_lowest_mu"] > temp_results['expected_mu']: outputs["expected_lowest_mu"] = temp_results['expected_mu'] return outputs From 977bd2055c4a1006737baae5990784848e4bc606 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 13:56:14 +0800 Subject: [PATCH 06/26] fix pylint --- .../metis_tuner/lib_acquisition_function.py | 50 ++++++++++++------- .../metis_tuner/lib_constraint_summation.py | 37 ++++++++------ src/sdk/pynni/nni/metis_tuner/lib_data.py | 9 ++-- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py b/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py index 8beff1a6e6..476323c93f 100644 --- a/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py +++ b/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py @@ -16,7 +16,11 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +""" +lib_acquisition_function.py +""" import sys import numpy @@ -33,9 +37,9 @@ def next_hyperparameter_expected_improvement(fun_prediction, samples_y_aggregation, minimize_starting_points, minimize_constraints_fun=None): - ''' + """ "Expected Improvement" acquisition function - ''' + """ best_x = None best_acquisition_value = None x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] @@ -70,6 +74,7 @@ def next_hyperparameter_expected_improvement(fun_prediction, return outputs + def _expected_improvement(x, fun_prediction, fun_prediction_args, x_bounds, x_types, samples_y_aggregation, minimize_constraints_fun): @@ -77,7 +82,8 @@ def _expected_improvement(x, fun_prediction, fun_prediction_args, x = lib_data.match_val_type(x, x_bounds, x_types) expected_improvement = sys.maxsize - if (minimize_constraints_fun is None) or (minimize_constraints_fun(x) is True): + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): mu, sigma = fun_prediction(x, *fun_prediction_args) loss_optimum = min(samples_y_aggregation) @@ -87,7 +93,7 @@ def _expected_improvement(x, fun_prediction, fun_prediction_args, with numpy.errstate(divide="ignore"): Z = scaling_factor * (mu - loss_optimum) / sigma expected_improvement = scaling_factor * (mu - loss_optimum) * \ - norm.cdf(Z) + sigma * norm.pdf(Z) + norm.cdf(Z) + sigma * norm.pdf(Z) expected_improvement = 0.0 if sigma == 0.0 else expected_improvement # We want expected_improvement to be as large as possible @@ -101,9 +107,9 @@ def next_hyperparameter_lowest_confidence(fun_prediction, x_bounds, x_types, minimize_starting_points, minimize_constraints_fun=None): - ''' + """ "Lowest Confidence" acquisition function - ''' + """ best_x = None best_acquisition_value = None x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] @@ -120,10 +126,12 @@ def next_hyperparameter_lowest_confidence(fun_prediction, x_types, minimize_constraints_fun)) - if (best_acquisition_value) is None or (res.fun < best_acquisition_value): + if (best_acquisition_value) is None or ( + res.fun < best_acquisition_value): res.x = numpy.ndarray.tolist(res.x) res.x = lib_data.match_val_type(res.x, x_bounds, x_types) - if (minimize_constraints_fun is None) or (minimize_constraints_fun(res.x) is True): + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): best_acquisition_value = res.fun best_x = res.x @@ -134,13 +142,15 @@ def next_hyperparameter_lowest_confidence(fun_prediction, 'expected_sigma': sigma, 'acquisition_func': "lc"} return outputs + def _lowest_confidence(x, fun_prediction, fun_prediction_args, x_bounds, x_types, minimize_constraints_fun): # This is only for step-wise optimization x = lib_data.match_val_type(x, x_bounds, x_types) ci = sys.maxsize - if (minimize_constraints_fun is None) or (minimize_constraints_fun(x) is True): + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): mu, sigma = fun_prediction(x, *fun_prediction_args) ci = (sigma * 1.96 * 2) / mu # We want ci to be as large as possible @@ -156,9 +166,9 @@ def next_hyperparameter_lowest_mu(fun_prediction, x_bounds, x_types, minimize_starting_points, minimize_constraints_fun=None): - ''' + """ "Lowest Mu" acquisition function - ''' + """ best_x = None best_acquisition_value = None x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] @@ -169,13 +179,15 @@ def next_hyperparameter_lowest_mu(fun_prediction, x0=starting_point.reshape(1, -1), bounds=x_bounds_minmax, method="L-BFGS-B", - args=(fun_prediction, fun_prediction_args, \ + args=(fun_prediction, fun_prediction_args, x_bounds, x_types, minimize_constraints_fun)) - if (best_acquisition_value is None) or (res.fun < best_acquisition_value): + if (best_acquisition_value is None) or ( + res.fun < best_acquisition_value): res.x = numpy.ndarray.tolist(res.x) res.x = lib_data.match_val_type(res.x, x_bounds, x_types) - if (minimize_constraints_fun is None) or (minimize_constraints_fun(res.x) is True): + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): best_acquisition_value = res.fun best_x = res.x @@ -189,14 +201,14 @@ def next_hyperparameter_lowest_mu(fun_prediction, def _lowest_mu(x, fun_prediction, fun_prediction_args, x_bounds, x_types, minimize_constraints_fun): - ''' + """ Calculate the lowest mu - ''' + """ # This is only for step-wise optimization x = lib_data.match_val_type(x, x_bounds, x_types) mu = sys.maxsize - if (minimize_constraints_fun is None) or (minimize_constraints_fun(x) is True): + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): mu, _ = fun_prediction(x, *fun_prediction_args) return mu - \ No newline at end of file diff --git a/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py b/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py index 1e9daaee95..cc385e9afc 100644 --- a/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py +++ b/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py @@ -16,7 +16,11 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +""" +lib_constraint_summation.py +""" import math import random @@ -39,6 +43,7 @@ def check_feasibility(x_bounds, lowerbound, upperbound): return (x_bounds_lowerbound <= lowerbound <= x_bounds_upperbound) or \ (x_bounds_lowerbound <= upperbound <= x_bounds_upperbound) + def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): ''' Key idea is that we try to move towards upperbound, by randomly choose one @@ -55,7 +60,8 @@ def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): if x_types[i] == "discrete_int": x_idx_sorted.append([i, len(x_bounds[i])]) elif (x_types[i] == "range_int") or (x_types[i] == "range_continuous"): - x_idx_sorted.append([i, math.floor(x_bounds[i][1] - x_bounds[i][0])]) + x_idx_sorted.append( + [i, math.floor(x_bounds[i][1] - x_bounds[i][0])]) x_idx_sorted = sorted(x_idx_sorted, key=itemgetter(1)) for _ in range(max_retries): @@ -77,12 +83,13 @@ def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): temp.append(j) # Randomly pick a number from the integer array if temp: - outputs[x_idx] = temp[random.randint(0, len(temp) - 1)] + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] elif (x_types[x_idx] == "range_int") or \ - (x_types[x_idx] == "range_continuous"): - outputs[x_idx] = random.randint(x_bounds[x_idx][0], - min(x_bounds[x_idx][-1], budget_max)) + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + x_bounds[x_idx][0], min(x_bounds[x_idx][-1], budget_max)) else: # The last x that we need to assign a random number @@ -91,26 +98,28 @@ def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): # This check: # is our smallest possible value going to overflow the available budget space, - # and is our largest possible value going to underflow the lower bound + # and is our largest possible value going to underflow the + # lower bound if (x_bounds[x_idx][0] <= budget_max) and \ (x_bounds[x_idx][-1] >= randint_lowerbound): if x_types[x_idx] == "discrete_int": temp = [] for j in x_bounds[x_idx]: - # if (j <= budget_max) and (j >= randint_lowerbound): + # if (j <= budget_max) and (j >= + # randint_lowerbound): if randint_lowerbound <= j <= budget_max: temp.append(j) if temp: - outputs[x_idx] = temp[random.randint(0, len(temp) - 1)] + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] elif (x_types[x_idx] == "range_int") or \ (x_types[x_idx] == "range_continuous"): - outputs[x_idx] = random.randint(randint_lowerbound, - min(x_bounds[x_idx][1], budget_max)) + outputs[x_idx] = random.randint( + randint_lowerbound, min( + x_bounds[x_idx][1], budget_max)) if outputs[x_idx] is None: break - else: - budget_allocated += outputs[x_idx] + budget_allocated += outputs[x_idx] if None not in outputs: break return outputs - \ No newline at end of file diff --git a/src/sdk/pynni/nni/metis_tuner/lib_data.py b/src/sdk/pynni/nni/metis_tuner/lib_data.py index c42f3ee735..c7e4c0d668 100644 --- a/src/sdk/pynni/nni/metis_tuner/lib_data.py +++ b/src/sdk/pynni/nni/metis_tuner/lib_data.py @@ -16,7 +16,8 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. import math import random @@ -31,7 +32,8 @@ def match_val_type(vals, vals_bounds, vals_types): for i, _ in enumerate(vals_types): if vals_types[i] == "discrete_int": # Find the closest integer in the array, vals_bounds - vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) + vals_new.append( + min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) elif vals_types[i] == "range_int": # Round down to the nearest integer vals_new.append(math.floor(vals[i])) @@ -55,7 +57,7 @@ def rand(x_bounds, x_types): temp = x_bounds[i][random.randint(0, len(x_bounds[i]) - 1)] outputs.append(temp) elif x_types[i] == "range_int": - temp = random.randint(x_bounds[i][0], x_bounds[i][1] -1) + temp = random.randint(x_bounds[i][0], x_bounds[i][1] - 1) outputs.append(temp) elif x_types[i] == "range_continuous": temp = random.uniform(x_bounds[i][0], x_bounds[i][1]) @@ -64,4 +66,3 @@ def rand(x_bounds, x_types): return None return outputs - \ No newline at end of file From 49a5490171e9ceadda72d088813f3a9bb6806c2e Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 14:00:34 +0800 Subject: [PATCH 07/26] update --- .../metis_tuner/Regression_GMM/CreateModel.py | 16 ++++++---- .../metis_tuner/Regression_GMM/Selection.py | 29 ++++++++++++------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py index 3ed39e0cf8..7bc9e070fb 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py @@ -16,7 +16,8 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. import os import sys @@ -31,7 +32,8 @@ def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): ''' Create the Gaussian Mixture Model ''' - samples = [samples_x[i] + [samples_y_aggregation[i]] for i in range(0, len(samples_x))] + samples = [samples_x[i] + [samples_y_aggregation[i]] + for i in range(0, len(samples_x))] # Sorts so that we can get the top samples samples = sorted(samples, key=itemgetter(-1)) @@ -39,13 +41,16 @@ def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): samples_goodbatch = samples[0:samples_goodbatch_size] samples_badbatch = samples[samples_goodbatch_size:] - samples_x_goodbatch = [sample_goodbatch[0:-1] for sample_goodbatch in samples_goodbatch] + samples_x_goodbatch = [sample_goodbatch[0:-1] + for sample_goodbatch in samples_goodbatch] #samples_y_goodbatch = [sample_goodbatch[-1] for sample_goodbatch in samples_goodbatch] - samples_x_badbatch = [sample_badbatch[0:-1] for sample_badbatch in samples_badbatch] + samples_x_badbatch = [sample_badbatch[0:-1] + for sample_badbatch in samples_badbatch] # === Trains GMM clustering models === # #sys.stderr.write("[%s] Train GMM's GMM model\n" % (os.path.basename(__file__))) - bgmm_goodbatch = mm.BayesianGaussianMixture(n_components=max(1, samples_goodbatch_size - 1)) + bgmm_goodbatch = mm.BayesianGaussianMixture( + n_components=max(1, samples_goodbatch_size - 1)) bad_n_components = max(1, len(samples_x) - samples_goodbatch_size - 1) bgmm_badbatch = mm.BayesianGaussianMixture(n_components=bad_n_components) bgmm_goodbatch.fit(samples_x_goodbatch) @@ -55,4 +60,3 @@ def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): model['clusteringmodel_good'] = bgmm_goodbatch model['clusteringmodel_bad'] = bgmm_badbatch return model - \ No newline at end of file diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py index 9341e49e2b..d68d1920c5 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py @@ -16,7 +16,8 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. import os import random @@ -24,7 +25,6 @@ import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation -import nni.metis_tuner.lib_data as lib_data sys.path.insert(1, os.path.join(sys.path[0], '..')) @@ -34,14 +34,17 @@ CONSTRAINT_PARAMS_IDX = [] -def _ratio_scores(parameters_value, clusteringmodel_gmm_good, clusteringmodel_gmm_bad): +def _ratio_scores(parameters_value, clusteringmodel_gmm_good, + clusteringmodel_gmm_bad): ''' The ratio is smaller the better ''' - ratio = clusteringmodel_gmm_good.score([parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value]) + ratio = clusteringmodel_gmm_good.score( + [parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value]) sigma = 0 return ratio, sigma + def selection_r(x_bounds, x_types, clusteringmodel_gmm_good, @@ -51,16 +54,18 @@ def selection_r(x_bounds, ''' Select using different types. ''' - minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) - + minimize_starting_points = clusteringmodel_gmm_good.sample( + n_samples=num_starting_points) + outputs = selection(x_bounds, x_types, clusteringmodel_gmm_good, clusteringmodel_gmm_bad, minimize_starting_points[0], minimize_constraints_fun) - + return outputs + def selection(x_bounds, x_types, clusteringmodel_gmm_good, @@ -70,13 +75,14 @@ def selection(x_bounds, ''' Select the lowest mu value ''' - results = lib_acquisition_function.next_hyperparameter_lowest_mu(\ - _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad],\ - x_bounds, x_types, minimize_starting_points, \ - minimize_constraints_fun=minimize_constraints_fun) + results = lib_acquisition_function.next_hyperparameter_lowest_mu( + _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad], + x_bounds, x_types, minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) return results + def _rand_with_constraints(x_bounds, x_types): ''' Random generate the variable with constraints @@ -97,6 +103,7 @@ def _rand_with_constraints(x_bounds, x_types): outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) return outputs + def _minimize_constraints_fun_summation(x): ''' Minimize constraints fun summation From 4b26e05f51689d6ca545de9a35028398345ac35d Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 14:06:51 +0800 Subject: [PATCH 08/26] fix pylint in metis_tuner --- .../Regression_GP/OutlierDetection.py | 54 +++++++++++-------- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 8 +++ 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py index 95a8abfdc6..864ee6e2fe 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py @@ -16,15 +16,18 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +""" +OutlierDectection.py +""" - -import argparse, json, os, sys +import os +import sys from multiprocessing.dummy import Pool as ThreadPool import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model import nni.metis_tuner.Regression_GP.Prediction as gp_prediction -import nni.metis_tuner.lib_data as lib_data sys.path.insert(1, os.path.join(sys.path[0], '..')) @@ -34,15 +37,17 @@ def _outlierDetection_threaded(inputs): Detect the outlier ''' [samples_idx, samples_x, samples_y_aggregation] = inputs - sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n"\ - % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n" + % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) outlier = None - # Create a diagnostic regression model which removes the sample that we want to evaluate - diagnostic_regressor_gp = gp_create_model.create_model(\ - samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ - samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) - mu, sigma = gp_prediction.predict(samples_x[samples_idx], diagnostic_regressor_gp['model']) + # Create a diagnostic regression model which removes the sample that we + # want to evaluate + diagnostic_regressor_gp = gp_create_model.create_model( + samples_x[0:samples_idx] + samples_x[samples_idx + 1:], + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict( + samples_x[samples_idx], diagnostic_regressor_gp['model']) # 2.33 is the z-score for 98% confidence level if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): @@ -52,16 +57,18 @@ def _outlierDetection_threaded(inputs): "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)} return outlier + def outlierDetection_threaded(samples_x, samples_y_aggregation): ''' Use Multi-thread to detect the outlier ''' outliers = [] - threads_inputs = [[samples_idx, samples_x, samples_y_aggregation]\ - for samples_idx in range(0, len(samples_x))] + threads_inputs = [[samples_idx, samples_x, samples_y_aggregation] + for samples_idx in range(0, len(samples_x))] threads_pool = ThreadPool(min(4, len(threads_inputs))) - threads_results = threads_pool.map(_outlierDetection_threaded, threads_inputs) + threads_results = threads_pool.map( + _outlierDetection_threaded, threads_inputs) threads_pool.close() threads_pool.join() @@ -69,21 +76,21 @@ def outlierDetection_threaded(samples_x, samples_y_aggregation): if threads_result is not None: outliers.append(threads_result) else: - print("error here.") + sys.stderr.write("Error: threads_result is None.") outliers = None if len(outliers) == 0 else outliers return outliers + def outlierDetection(samples_x, samples_y_aggregation): - ''' - ''' outliers = [] for samples_idx in range(0, len(samples_x)): - #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" - # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) - diagnostic_regressor_gp = gp_create_model.create_model(\ - samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ - samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" \ + % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + + diagnostic_regressor_gp = gp_create_model.create_model( + samples_x[0:samples_idx] + samples_x[samples_idx + 1:], + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) mu, sigma = gp_prediction.predict(samples_x[samples_idx], diagnostic_regressor_gp['model']) # 2.33 is the z-score for 98% confidence level @@ -91,7 +98,8 @@ def outlierDetection(samples_x, samples_y_aggregation): outliers.append({"samples_idx": samples_idx, "expected_mu": mu, "expected_sigma": sigma, - "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) + "difference": \ + abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) outliers = None if len(outliers) == 0 else outliers return outliers diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index eec66956be..a55e1ff185 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -151,6 +151,7 @@ def __init__( self.x_bounds = None self.x_types = None + def update_search_space(self, search_space): """ Update the self.x_bounds and self.x_types by the search_space.json @@ -217,6 +218,7 @@ def update_search_space(self, search_space): self.minimize_starting_points = _rand_init( self.x_bounds, self.x_types, self.selection_num_starting_points) + def _pack_output(self, init_parameter): """ Pack the output @@ -232,8 +234,10 @@ def _pack_output(self, init_parameter): output = {} for i, param in enumerate(init_parameter): output[self.key_order[i]] = param + return output + def generate_parameters(self, parameter_id, **kwargs): """ Generate next parameter for trial @@ -272,6 +276,7 @@ def generate_parameters(self, parameter_id, **kwargs): logger.info("Generate paramageters: \n%s", str(results)) return results + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): """ Tuner receive result from trial. @@ -279,7 +284,9 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): Parameters ---------- parameter_id : int + The id of parameters, generated by nni manager. parameters : dict + A group of parameters that trial has tried. value : dict/float if value is dict, it should have "default" key. """ @@ -315,6 +322,7 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): # calculate y aggregation self.samples_y_aggregation.append([value]) + def _selection( self, samples_x, From bbe6cbc54f47b7c6a789eb571072e608e5f323e4 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Tue, 29 Oct 2019 14:21:08 +0800 Subject: [PATCH 09/26] update in networkmorphsim_tuner --- .../nni/networkmorphism_tuner/bayesian.py | 26 ++++-- .../pynni/nni/networkmorphism_tuner/graph.py | 65 +++++++++----- .../graph_transformer.py | 20 +++-- .../layer_transformer.py | 24 ++++-- .../pynni/nni/networkmorphism_tuner/layers.py | 56 ++++++++++-- .../networkmorphism_tuner.py | 85 +++++++++++++++---- src/sdk/pynni/nni/networkmorphism_tuner/nn.py | 33 +++++-- .../test_networkmorphism_tuner.py | 17 ++-- 8 files changed, 246 insertions(+), 80 deletions(-) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py b/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py index 360771139a..15c5e83cdd 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py @@ -38,7 +38,7 @@ def layer_distance(a, b): """The distance between two layers.""" # pylint: disable=unidiomatic-typecheck - if type(a) != type(b): + if not isinstance(a, type(b)): return 1.0 if is_layer(a, "Conv"): att_diff = [ @@ -96,7 +96,8 @@ def skip_connection_distance(a, b): return 1.0 len_a = abs(a[1] - a[0]) len_b = abs(b[1] - b[0]) - return (abs(a[0] - b[0]) + abs(len_a - len_b)) / (max(a[0], b[0]) + max(len_a, len_b)) + return (abs(a[0] - b[0]) + abs(len_a - len_b)) / \ + (max(a[0], b[0]) + max(len_a, len_b)) def skip_connections_distance(list_a, list_b): @@ -161,7 +162,8 @@ def fit(self, train_x, train_y): def incremental_fit(self, train_x, train_y): """ Incrementally fit the regressor. """ if not self._first_fitted: - raise ValueError("The first_fit function needs to be called first.") + raise ValueError( + "The first_fit function needs to be called first.") train_x, train_y = np.array(train_x), np.array(train_y) @@ -174,7 +176,7 @@ def incremental_fit(self, train_x, train_y): temp_distance_matrix = np.concatenate((up_k, down_k), axis=0) k_matrix = bourgain_embedding_matrix(temp_distance_matrix) diagonal = np.diag_indices_from(k_matrix) - diagonal = (diagonal[0][-len(train_x) :], diagonal[1][-len(train_x) :]) + diagonal = (diagonal[0][-len(train_x):], diagonal[1][-len(train_x):]) k_matrix[diagonal] += self.alpha try: @@ -186,7 +188,8 @@ def incremental_fit(self, train_x, train_y): self._y = np.concatenate((self._y, train_y), axis=0) self._distance_matrix = temp_distance_matrix - self._alpha_vector = cho_solve((self._l_matrix, True), self._y) # Line 3 + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 return self @@ -209,7 +212,8 @@ def first_fit(self, train_x, train_y): self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 - self._alpha_vector = cho_solve((self._l_matrix, True), self._y) # Line 3 + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 self._first_fitted = True return self @@ -227,7 +231,9 @@ def predict(self, train_x): # compute inverse K_inv of K based on its Cholesky # decomposition L and its inverse L_inv - l_inv = solve_triangular(self._l_matrix.T, np.eye(self._l_matrix.shape[0])) + l_inv = solve_triangular( + self._l_matrix.T, np.eye( + self._l_matrix.shape[0])) k_inv = l_inv.dot(l_inv.T) # Compute variance of predictive distribution y_var = np.ones(len(train_x), dtype=np.float) @@ -378,7 +384,11 @@ def generate(self, descriptors): continue temp_acq_value = self.acq(temp_graph) - pq.put(elem_class(temp_acq_value, elem.father_id, temp_graph)) + pq.put( + elem_class( + temp_acq_value, + elem.father_id, + temp_graph)) descriptors.append(temp_graph.extract_descriptor()) if self._accept_new_acq_value(opt_acq, temp_acq_value): opt_acq = temp_acq_value diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py b/src/sdk/pynni/nni/networkmorphism_tuner/graph.py index 82b37f7243..170f1a268c 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/graph.py @@ -249,7 +249,8 @@ def _redirect_edge(self, u_id, v_id, new_v_id): self.reverse_adj_list[v_id].remove(edge_tuple) break self.reverse_adj_list[new_v_id].append((u_id, layer_id)) - for index, value in enumerate(self.layer_id_to_output_node_ids[layer_id]): + for index, value in enumerate( + self.layer_id_to_output_node_ids[layer_id]): if value == v_id: self.layer_id_to_output_node_ids[layer_id][index] = new_v_id break @@ -350,7 +351,8 @@ def _search(self, u, start_dim, total_dim, n_add): self._replace_layer(layer_id, new_layer) elif is_layer(layer, "BatchNormalization"): - new_layer = wider_bn(layer, start_dim, total_dim, n_add, self.weighted) + new_layer = wider_bn( + layer, start_dim, total_dim, n_add, self.weighted) self._replace_layer(layer_id, new_layer) self._search(v, start_dim, total_dim, n_add) @@ -405,7 +407,8 @@ def to_deeper_model(self, target_id, new_layer): target_id: A convolutional layer ID. The new block should be inserted after the block. new_layer: An instance of StubLayer subclasses. """ - self.operation_history.append(("to_deeper_model", target_id, new_layer)) + self.operation_history.append( + ("to_deeper_model", target_id, new_layer)) input_id = self.layer_id_to_input_node_ids[target_id][0] output_id = self.layer_id_to_output_node_ids[target_id][0] if self.weighted: @@ -478,14 +481,20 @@ def to_add_skip_model(self, start_id, end_id): pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] end_node_id = self.layer_id_to_output_node_ids[end_id][0] - skip_output_id = self._insert_pooling_layer_chain(start_node_id, end_node_id) + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) # Add the conv layer - new_conv_layer = get_conv_class(self.n_dim)(filters_start, filters_end, 1) + new_conv_layer = get_conv_class( + self.n_dim)( + filters_start, + filters_end, + 1) skip_output_id = self.add_layer(new_conv_layer, skip_output_id) # Add the add layer. - add_input_node_id = self._add_node(deepcopy(self.node_list[end_node_id])) + add_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) add_layer = StubAdd() self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id) @@ -504,7 +513,8 @@ def to_add_skip_model(self, start_id, end_id): weights = np.zeros((filters_end, filters_start) + filter_shape) bias = np.zeros(filters_end) new_conv_layer.set_weights( - (add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))) + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) ) def to_concat_skip_model(self, start_id, end_id): @@ -513,7 +523,8 @@ def to_concat_skip_model(self, start_id, end_id): start_id: The convolutional layer ID, after which to start the skip-connection. end_id: The convolutional layer ID, after which to end the skip-connection. """ - self.operation_history.append(("to_concat_skip_model", start_id, end_id)) + self.operation_history.append( + ("to_concat_skip_model", start_id, end_id)) filters_end = self.layer_list[end_id].output.shape[-1] filters_start = self.layer_list[start_id].output.shape[-1] start_node_id = self.layer_id_to_output_node_ids[start_id][0] @@ -521,9 +532,11 @@ def to_concat_skip_model(self, start_id, end_id): pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] end_node_id = self.layer_id_to_output_node_ids[end_id][0] - skip_output_id = self._insert_pooling_layer_chain(start_node_id, end_node_id) + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) - concat_input_node_id = self._add_node(deepcopy(self.node_list[end_node_id])) + concat_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id) concat_layer = StubConcatenate() @@ -532,7 +545,10 @@ def to_concat_skip_model(self, start_id, end_id): self.node_list[skip_output_id], ] concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) - self._add_edge(concat_layer, concat_input_node_id, concat_output_node_id) + self._add_edge( + concat_layer, + concat_input_node_id, + concat_output_node_id) self._add_edge(concat_layer, skip_output_id, concat_output_node_id) concat_layer.output = self.node_list[concat_output_node_id] self.node_list[concat_output_node_id].shape = concat_layer.output_shape @@ -559,7 +575,8 @@ def to_concat_skip_model(self, start_id, end_id): ) bias = np.zeros(filters_end) new_conv_layer.set_weights( - (add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))) + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) ) def _insert_pooling_layer_chain(self, start_node_id, end_node_id): @@ -568,7 +585,8 @@ def _insert_pooling_layer_chain(self, start_node_id, end_node_id): new_layer = deepcopy(layer) if is_layer(new_layer, "Conv"): filters = self.node_list[start_node_id].shape[-1] - new_layer = get_conv_class(self.n_dim)(filters, filters, 1, layer.stride) + new_layer = get_conv_class(self.n_dim)( + filters, filters, 1, layer.stride) if self.weighted: init_conv_weight(new_layer) else: @@ -601,8 +619,10 @@ def extract_descriptor(self): temp_v = v temp_layer_id = layer_id skip_type = None - while not (temp_v in index_in_main_chain and temp_u in index_in_main_chain): - if is_layer(self.layer_list[temp_layer_id], "Concatenate"): + while not ( + temp_v in index_in_main_chain and temp_u in index_in_main_chain): + if is_layer( + self.layer_list[temp_layer_id], "Concatenate"): skip_type = NetworkDescriptor.CONCAT_CONNECT if is_layer(self.layer_list[temp_layer_id], "Add"): skip_type = NetworkDescriptor.ADD_CONNECT @@ -711,7 +731,8 @@ def deep_layer_ids(self): def wide_layer_ids(self): return ( - self._conv_layer_ids_in_order()[:-1] + self._dense_layer_ids_in_order()[:-1] + self._conv_layer_ids_in_order( + )[:-1] + self._dense_layer_ids_in_order()[:-1] ) def skip_connection_layer_ids(self): @@ -810,7 +831,8 @@ def __init__(self, graph): topo_node_list = self.graph.topological_order output_id = topo_node_list[-1] input_id = topo_node_list[0] - input_tensor = keras.layers.Input(shape=graph.node_list[input_id].shape) + input_tensor = keras.layers.Input( + shape=graph.node_list[input_id].shape) node_list = deepcopy(self.graph.node_list) node_list[input_id] = input_tensor @@ -838,7 +860,8 @@ def __init__(self, graph): output_tensor = keras.layers.Activation("softmax", name="activation_add")( output_tensor ) - self.model = keras.models.Model(inputs=input_tensor, outputs=output_tensor) + self.model = keras.models.Model( + inputs=input_tensor, outputs=output_tensor) if graph.weighted: for index, layer in enumerate(self.layers): @@ -892,7 +915,8 @@ def __init__(self, graph): for layer_id, item in enumerate(graph.layer_list): layer = graph.layer_list[layer_id] - layer_information = layer_description_extractor(layer, graph.node_to_id) + layer_information = layer_description_extractor( + layer, graph.node_to_id) layer_list.append((layer_id, layer_information)) data["node_list"] = node_list @@ -939,7 +963,8 @@ def json_to_graph(json_model: str): graph.input_shape = input_shape vis = json_model["vis"] - graph.vis = {tuple(item): True for item in vis} if vis is not None else None + graph.vis = { + tuple(item): True for item in vis} if vis is not None else None graph.weighted = json_model["weighted"] layer_id_to_input_node_ids = json_model["layer_id_to_input_node_ids"] graph.layer_id_to_input_node_ids = { diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py b/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py index a318188f3e..6b36e8ed97 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py @@ -40,7 +40,8 @@ def to_wider_graph(graph): ''' weighted_layer_ids = graph.wide_layer_ids() weighted_layer_ids = list( - filter(lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids) + filter( + lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids) ) wider_layers = sample(weighted_layer_ids, 1) @@ -58,12 +59,14 @@ def to_wider_graph(graph): def to_skip_connection_graph(graph): ''' skip connection graph ''' - # The last conv layer cannot be widen since wider operator cannot be done over the two sides of flatten. + # The last conv layer cannot be widen since wider operator cannot be done + # over the two sides of flatten. weighted_layer_ids = graph.skip_connection_layer_ids() valid_connection = [] - for skip_type in sorted([NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]): + for skip_type in sorted( + [NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]): for index_a in range(len(weighted_layer_ids)): - for index_b in range(len(weighted_layer_ids))[index_a + 1 :]: + for index_b in range(len(weighted_layer_ids))[index_a + 1:]: valid_connection.append((index_a, index_b, skip_type)) if not valid_connection: @@ -84,9 +87,14 @@ def create_new_layer(layer, n_dim): input_shape = layer.output.shape dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU] - conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim), StubReLU] + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim), + StubReLU] if is_layer(layer, "ReLU"): - conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim)] + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim)] dense_deeper_classes = [StubDense, get_dropout_class(n_dim)] elif is_layer(layer, "Dropout"): dense_deeper_classes = [StubDense, StubReLU] diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py b/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py index 8775431e58..ac87744a4b 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py @@ -52,7 +52,8 @@ def deeper_conv_block(conv_layer, kernel_size, weighted=True): if weighted: new_conv_layer.set_weights( - (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1]))) + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) ) new_weights = [ add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), @@ -74,7 +75,8 @@ def dense_to_deeper_block(dense_layer, weighted=True): new_dense_layer = StubDense(units, units) if weighted: new_dense_layer.set_weights( - (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1]))) + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) ) return [StubReLU(), new_dense_layer] @@ -97,8 +99,11 @@ def wider_pre_dense(layer, n_add, weighted=True): teacher_index = rand[i] new_weight = teacher_w[teacher_index, :] new_weight = new_weight[np.newaxis, :] - student_w = np.concatenate((student_w, add_noise(new_weight, student_w)), axis=0) - student_b = np.append(student_b, add_noise(teacher_b[teacher_index], student_b)) + student_w = np.concatenate( + (student_w, add_noise(new_weight, student_w)), axis=0) + student_b = np.append( + student_b, add_noise( + teacher_b[teacher_index], student_b)) new_pre_layer = StubDense(layer.input_units, n_units2 + n_add) new_pre_layer.set_weights((student_w, student_b)) @@ -209,7 +214,7 @@ def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True): student_w[:, : start_dim * n_units_each_channel], add_noise(new_weight, student_w), student_w[ - :, start_dim * n_units_each_channel : total_dim * n_units_each_channel + :, start_dim * n_units_each_channel: total_dim * n_units_each_channel ], ), axis=1, @@ -225,7 +230,8 @@ def add_noise(weights, other_weights): ''' w_range = np.ptp(other_weights.flatten()) noise_range = NOISE_RATIO * w_range - noise = np.random.uniform(-noise_range / 2.0, noise_range / 2.0, weights.shape) + noise = np.random.uniform(-noise_range / 2.0, + noise_range / 2.0, weights.shape) return np.add(noise, weights) @@ -236,7 +242,8 @@ def init_dense_weight(layer): weight = np.eye(units) bias = np.zeros(units) layer.set_weights( - (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1]))) + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) ) @@ -256,7 +263,8 @@ def init_conv_weight(layer): bias = np.zeros(n_filters) layer.set_weights( - (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1]))) + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) ) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py index e174f87c5b..2b1ea1c9b7 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py @@ -30,6 +30,7 @@ class AvgPool(nn.Module): '''AvgPool Module. ''' + def __init__(self): super().__init__() @@ -41,6 +42,7 @@ def forward(self, input_tensor): class GlobalAvgPool1d(AvgPool): '''GlobalAvgPool1d Module. ''' + def forward(self, input_tensor): return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view( input_tensor.size()[:2] @@ -50,6 +52,7 @@ def forward(self, input_tensor): class GlobalAvgPool2d(AvgPool): '''GlobalAvgPool2d Module. ''' + def forward(self, input_tensor): return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view( input_tensor.size()[:2] @@ -59,6 +62,7 @@ def forward(self, input_tensor): class GlobalAvgPool3d(AvgPool): '''GlobalAvgPool3d Module. ''' + def forward(self, input_tensor): return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view( input_tensor.size()[:2] @@ -68,6 +72,7 @@ def forward(self, input_tensor): class StubLayer: '''StubLayer Module. Base Module. ''' + def __init__(self, input_node=None, output_node=None): self.input = input_node self.output = output_node @@ -133,9 +138,11 @@ def __str__(self): class StubWeightBiasLayer(StubLayer): '''StubWeightBiasLayer Module to set the bias. ''' + def import_weights(self, torch_layer): self.set_weights( - (torch_layer.weight.data.cpu().numpy(), torch_layer.bias.data.cpu().numpy()) + (torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy()) ) def import_weights_keras(self, keras_layer): @@ -152,6 +159,7 @@ def export_weights_keras(self, keras_layer): class StubBatchNormalization(StubWeightBiasLayer): '''StubBatchNormalization Module. Batch Norm. ''' + def __init__(self, num_features, input_node=None, output_node=None): super().__init__(input_node, output_node) self.num_features = num_features @@ -183,6 +191,7 @@ def to_real_layer(self): class StubBatchNormalization1d(StubBatchNormalization): '''StubBatchNormalization1d Module. ''' + def to_real_layer(self): return torch.nn.BatchNorm1d(self.num_features) @@ -190,6 +199,7 @@ def to_real_layer(self): class StubBatchNormalization2d(StubBatchNormalization): '''StubBatchNormalization2d Module. ''' + def to_real_layer(self): return torch.nn.BatchNorm2d(self.num_features) @@ -197,6 +207,7 @@ def to_real_layer(self): class StubBatchNormalization3d(StubBatchNormalization): '''StubBatchNormalization3d Module. ''' + def to_real_layer(self): return torch.nn.BatchNorm3d(self.num_features) @@ -204,6 +215,7 @@ def to_real_layer(self): class StubDense(StubWeightBiasLayer): '''StubDense Module. Linear. ''' + def __init__(self, input_units, units, input_node=None, output_node=None): super().__init__(input_node, output_node) self.input_units = input_units @@ -214,7 +226,9 @@ def output_shape(self): return (self.units,) def import_weights_keras(self, keras_layer): - self.set_weights((keras_layer.get_weights()[0].T, keras_layer.get_weights()[1])) + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) def export_weights_keras(self, keras_layer): keras_layer.set_weights((self.weights[0].T, self.weights[1])) @@ -229,7 +243,9 @@ def to_real_layer(self): class StubConv(StubWeightBiasLayer): '''StubConv Module. Conv. ''' - def __init__(self, input_channel, filters, kernel_size, stride=1, input_node=None, output_node=None): + + def __init__(self, input_channel, filters, kernel_size, + stride=1, input_node=None, output_node=None): super().__init__(input_node, output_node) self.input_channel = input_channel self.filters = filters @@ -248,13 +264,16 @@ def output_shape(self): return tuple(ret) def import_weights_keras(self, keras_layer): - self.set_weights((keras_layer.get_weights()[0].T, keras_layer.get_weights()[1])) + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) def export_weights_keras(self, keras_layer): keras_layer.set_weights((self.weights[0].T, self.weights[1])) def size(self): - return (self.input_channel * self.kernel_size * self.kernel_size + 1) * self.filters + return (self.input_channel * self.kernel_size * + self.kernel_size + 1) * self.filters @abstractmethod def to_real_layer(self): @@ -280,6 +299,7 @@ def __str__(self): class StubConv1d(StubConv): '''StubConv1d Module. ''' + def to_real_layer(self): return torch.nn.Conv1d( self.input_channel, @@ -293,6 +313,7 @@ def to_real_layer(self): class StubConv2d(StubConv): '''StubConv2d Module. ''' + def to_real_layer(self): return torch.nn.Conv2d( self.input_channel, @@ -306,6 +327,7 @@ def to_real_layer(self): class StubConv3d(StubConv): '''StubConv3d Module. ''' + def to_real_layer(self): return torch.nn.Conv3d( self.input_channel, @@ -319,6 +341,7 @@ def to_real_layer(self): class StubAggregateLayer(StubLayer): '''StubAggregateLayer Module. ''' + def __init__(self, input_nodes=None, output_node=None): if input_nodes is None: input_nodes = [] @@ -368,6 +391,7 @@ def to_real_layer(self): class StubReLU(StubLayer): '''StubReLU Module. ''' + def to_real_layer(self): return torch.nn.ReLU() @@ -375,6 +399,7 @@ def to_real_layer(self): class StubSoftmax(StubLayer): '''StubSoftmax Module. ''' + def to_real_layer(self): return torch.nn.LogSoftmax(dim=1) @@ -382,6 +407,7 @@ def to_real_layer(self): class StubDropout(StubLayer): '''StubDropout Module. ''' + def __init__(self, rate, input_node=None, output_node=None): super().__init__(input_node, output_node) self.rate = rate @@ -394,6 +420,7 @@ def to_real_layer(self): class StubDropout1d(StubDropout): '''StubDropout1d Module. ''' + def to_real_layer(self): return torch.nn.Dropout(self.rate) @@ -401,6 +428,7 @@ def to_real_layer(self): class StubDropout2d(StubDropout): '''StubDropout2d Module. ''' + def to_real_layer(self): return torch.nn.Dropout2d(self.rate) @@ -408,6 +436,7 @@ def to_real_layer(self): class StubDropout3d(StubDropout): '''StubDropout3d Module. ''' + def to_real_layer(self): return torch.nn.Dropout3d(self.rate) @@ -415,6 +444,7 @@ def to_real_layer(self): class StubInput(StubLayer): '''StubInput Module. ''' + def __init__(self, input_node=None, output_node=None): super().__init__(input_node, output_node) @@ -460,6 +490,7 @@ def to_real_layer(self): class StubPooling2d(StubPooling): '''StubPooling2d Module. ''' + def to_real_layer(self): return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride) @@ -467,6 +498,7 @@ def to_real_layer(self): class StubPooling3d(StubPooling): '''StubPooling3d Module. ''' + def to_real_layer(self): return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride) @@ -474,6 +506,7 @@ def to_real_layer(self): class StubGlobalPooling(StubLayer): '''StubGlobalPooling Module. ''' + def __init__(self, input_node=None, output_node=None): super().__init__(input_node, output_node) @@ -489,6 +522,7 @@ def to_real_layer(self): class StubGlobalPooling1d(StubGlobalPooling): '''StubGlobalPooling1d Module. ''' + def to_real_layer(self): return GlobalAvgPool1d() @@ -496,6 +530,7 @@ def to_real_layer(self): class StubGlobalPooling2d(StubGlobalPooling): '''StubGlobalPooling2d Module. ''' + def to_real_layer(self): return GlobalAvgPool2d() @@ -503,6 +538,7 @@ def to_real_layer(self): class StubGlobalPooling3d(StubGlobalPooling): '''StubGlobalPooling3d Module. ''' + def to_real_layer(self): return GlobalAvgPool3d() @@ -510,6 +546,7 @@ def to_real_layer(self): class TorchConcatenate(nn.Module): '''TorchConcatenate Module. ''' + def forward(self, input_list): return torch.cat(input_list, dim=1) @@ -517,6 +554,7 @@ def forward(self, input_list): class TorchAdd(nn.Module): '''TorchAdd Module. ''' + def forward(self, input_list): return input_list[0] + input_list[1] @@ -524,9 +562,11 @@ def forward(self, input_list): class TorchFlatten(nn.Module): '''TorchFlatten Module. ''' + def forward(self, input_tensor): return input_tensor.view(input_tensor.size(0), -1) + def keras_dropout(layer, rate): '''keras dropout layer. ''' @@ -645,7 +685,8 @@ def layer_description_extractor(layer, node_to_id): layer.units, ] elif isinstance(layer, (StubBatchNormalization,)): - return (type(layer).__name__, layer_input, layer_output, layer.num_features) + return (type(layer).__name__, layer_input, + layer_output, layer.num_features) elif isinstance(layer, (StubDropout,)): return (type(layer).__name__, layer_input, layer_output, layer.rate) elif isinstance(layer, StubPooling): @@ -695,7 +736,8 @@ def layer_description_builder(layer_information, id_to_node): kernel_size = layer_information[3] stride = layer_information[4] padding = layer_information[5] - return eval(layer_type)(kernel_size, stride, padding, layer_input, layer_output) + return eval(layer_type)(kernel_size, stride, + padding, layer_input, layer_output) else: return eval(layer_type)(layer_input, layer_output) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py index d92870acbd..2edeb0bb8e 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py @@ -17,11 +17,13 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # ================================================================================================== +""" +networkmorphsim_tuner.py +""" import logging import os - from nni.tuner import Tuner from nni.utils import OptimizeMode, extract_scalar_reward from nni.networkmorphism_tuner.bayesian import BayesianOptimizer @@ -34,7 +36,35 @@ class NetworkMorphismTuner(Tuner): - """NetworkMorphismTuner is a tuner which using network morphism techniques.""" + """ + NetworkMorphismTuner is a tuner which using network morphism techniques. + + Attributes + ---------- + n_classes : int + The class number or output node number (default: {10}) + input_shape : tuple + A tuple including: (input_width, input_width, input_channel) + t_min : float + The minimum temperature for simulated annealing. (default: {Constant.T_MIN}) + beta : float + The beta in acquisition function. (default: {Constant.BETA}) + algorithm_name : str + algorithm name used in the network morphism (default: {"Bayesian"}) + optimize_mode : str + optimize mode "minimize" or "maximize" (default: {"minimize"}) + verbose : bool + verbose to print the log (default: {True}) + bo : BayesianOptimizer + The optimizer used in networkmorphsim tuner. + max_model_size : int + max model size to the graph (default: {Constant.MAX_MODEL_SIZE}) + default_model_len : int + default model length (default: {Constant.MODEL_LEN}) + default_model_width : int + default model width (default: {Constant.MODEL_WIDTH}) + search_space : dict + """ def __init__( self, @@ -52,7 +82,8 @@ def __init__( default_model_len=Constant.MODEL_LEN, default_model_width=Constant.MODEL_WIDTH, ): - """ initilizer of the NetworkMorphismTuner. + """ + initilizer of the NetworkMorphismTuner. Parameters ---------- @@ -92,7 +123,8 @@ def __init__( elif task == "common": self.generators = [MlpGenerator] else: - raise NotImplementedError('{} task not supported in List ["cv","common"]') + raise NotImplementedError( + '{} task not supported in List ["cv","common"]') self.n_classes = n_output_node self.input_shape = (input_width, input_width, input_channel) @@ -106,7 +138,8 @@ def __init__( self.verbose = verbose self.model_count = 0 - self.bo = BayesianOptimizer(self, self.t_min, self.optimize_mode, self.beta) + self.bo = BayesianOptimizer( + self, self.t_min, self.optimize_mode, self.beta) self.training_queue = [] self.descriptors = [] self.history = [] @@ -117,6 +150,7 @@ def __init__( self.search_space = dict() + def update_search_space(self, search_space): """ Update search space definition in tuner by search_space in neural architecture. @@ -140,7 +174,8 @@ def generate_parameters(self, parameter_id, **kwargs): new_father_id, generated_graph = self.generate() new_model_id = self.model_count self.model_count += 1 - self.training_queue.append((generated_graph, new_father_id, new_model_id)) + self.training_queue.append( + (generated_graph, new_father_id, new_model_id)) self.descriptors.append(generated_graph.extract_descriptor()) graph, father_id, model_id = self.training_queue.pop(0) @@ -153,12 +188,15 @@ def generate_parameters(self, parameter_id, **kwargs): return json_out def receive_trial_result(self, parameter_id, parameters, value, **kwargs): - """ Record an observation of the objective function. + """ + Record an observation of the objective function. Parameters ---------- parameter_id : int + the id of a group of paramters that generated by nni manager. parameters : dict + A group of parameters. value : dict/float if value is dict, it should have "default" key. """ @@ -175,8 +213,11 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): self.add_model(reward, model_id) self.update(father_id, graph, reward, model_id) + def init_search(self): - """Call the generators to generate the initial architectures for the search.""" + """ + Call the generators to generate the initial architectures for the search. + """ if self.verbose: logger.info("Initializing search.") for generator in self.generators: @@ -191,8 +232,10 @@ def init_search(self): if self.verbose: logger.info("Initialization finished.") + def generate(self): - """Generate the next neural architecture. + """ + Generate the next neural architecture. Returns ------- @@ -211,7 +254,8 @@ def generate(self): return new_father_id, generated_graph def update(self, other_info, graph, metric_value, model_id): - """ Update the controller with evaluation result of a neural architecture. + """ + Update the controller with evaluation result of a neural architecture. Parameters ---------- @@ -228,7 +272,8 @@ def update(self, other_info, graph, metric_value, model_id): self.bo.add_child(father_id, model_id) def add_model(self, metric_value, model_id): - """ Add model to the history, x_queue and y_queue + """ + Add model to the history, x_queue and y_queue Parameters ---------- @@ -252,16 +297,21 @@ def add_model(self, metric_value, model_id): file.close() return ret + def get_best_model_id(self): - """ Get the best model_id from history using the metric value + """ + Get the best model_id from history using the metric value """ if self.optimize_mode is OptimizeMode.Maximize: - return max(self.history, key=lambda x: x["metric_value"])["model_id"] + return max(self.history, key=lambda x: x["metric_value"])[ + "model_id"] return min(self.history, key=lambda x: x["metric_value"])["model_id"] + def load_model_by_id(self, model_id): - """Get the model by model_id + """ + Get the model by model_id Parameters ---------- @@ -281,7 +331,8 @@ def load_model_by_id(self, model_id): return load_model def load_best_model(self): - """ Get the best model by model id + """ + Get the best model by model id Returns ------- @@ -291,7 +342,8 @@ def load_best_model(self): return self.load_model_by_id(self.get_best_model_id()) def get_metric_value_by_id(self, model_id): - """ Get the model metric valud by its model_id + """ + Get the model metric valud by its model_id Parameters ---------- @@ -310,4 +362,3 @@ def get_metric_value_by_id(self, model_id): def import_data(self, data): pass - diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/nn.py b/src/sdk/pynni/nni/networkmorphism_tuner/nn.py index 363c06be5a..2e820ab2c2 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/nn.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/nn.py @@ -92,17 +92,25 @@ def generate(self, model_len=None, model_width=None): for i in range(model_len): output_node_id = graph.add_layer(StubReLU(), output_node_id) output_node_id = graph.add_layer( - self.batch_norm(graph.node_list[output_node_id].shape[-1]), output_node_id + self.batch_norm( + graph.node_list[output_node_id].shape[-1]), output_node_id ) output_node_id = graph.add_layer( - self.conv(temp_input_channel, model_width, kernel_size=3, stride=stride), + self.conv( + temp_input_channel, + model_width, + kernel_size=3, + stride=stride), output_node_id, ) temp_input_channel = model_width - if pooling_len == 0 or ((i + 1) % pooling_len == 0 and i != model_len - 1): - output_node_id = graph.add_layer(self.pooling(), output_node_id) + if pooling_len == 0 or ( + (i + 1) % pooling_len == 0 and i != model_len - 1): + output_node_id = graph.add_layer( + self.pooling(), output_node_id) - output_node_id = graph.add_layer(self.global_avg_pooling(), output_node_id) + output_node_id = graph.add_layer( + self.global_avg_pooling(), output_node_id) output_node_id = graph.add_layer( self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id ) @@ -111,7 +119,11 @@ def generate(self, model_len=None, model_width=None): output_node_id, ) output_node_id = graph.add_layer(StubReLU(), output_node_id) - graph.add_layer(StubDense(model_width, self.n_output_node), output_node_id) + graph.add_layer( + StubDense( + model_width, + self.n_output_node), + output_node_id) return graph @@ -145,7 +157,8 @@ def generate(self, model_len=None, model_width=None): if model_width is None: model_width = Constant.MODEL_WIDTH if isinstance(model_width, list) and not len(model_width) == model_len: - raise ValueError("The length of 'model_width' does not match 'model_len'") + raise ValueError( + "The length of 'model_width' does not match 'model_len'") elif isinstance(model_width, int): model_width = [model_width] * model_len @@ -162,5 +175,9 @@ def generate(self, model_len=None, model_width=None): output_node_id = graph.add_layer(StubReLU(), output_node_id) n_nodes_prev_layer = width - graph.add_layer(StubDense(n_nodes_prev_layer, self.n_output_node), output_node_id) + graph.add_layer( + StubDense( + n_nodes_prev_layer, + self.n_output_node), + output_node_id) return graph diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py index 09bbe820a9..5da56c487f 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py @@ -59,9 +59,12 @@ def test_graph_json_transform(self): graph_recover.layer_id_to_input_node_ids, ) self.assertEqual(graph_init.adj_list, graph_recover.adj_list) - self.assertEqual(graph_init.reverse_adj_list, graph_recover.reverse_adj_list) self.assertEqual( - len(graph_init.operation_history), len(graph_recover.operation_history) + graph_init.reverse_adj_list, + graph_recover.reverse_adj_list) + self.assertEqual( + len(graph_init.operation_history), len( + graph_recover.operation_history) ) self.assertEqual(graph_init.n_dim, graph_recover.n_dim) self.assertEqual(graph_init.conv, graph_recover.conv) @@ -71,7 +74,8 @@ def test_graph_json_transform(self): node_list_init = [node.shape for node in graph_init.node_list] node_list_recover = [node.shape for node in graph_recover.node_list] self.assertEqual(node_list_init, node_list_recover) - self.assertEqual(len(graph_init.node_to_id), len(graph_recover.node_to_id)) + self.assertEqual(len(graph_init.node_to_id), + len(graph_recover.node_to_id)) layer_list_init = [ layer_description_extractor(item, graph_init.node_to_id) for item in graph_init.layer_list @@ -82,7 +86,8 @@ def test_graph_json_transform(self): ] self.assertEqual(layer_list_init, layer_list_recover) - node_to_id_init = [graph_init.node_to_id[node] for node in graph_init.node_list] + node_to_id_init = [graph_init.node_to_id[node] + for node in graph_init.node_list] node_to_id_recover = [ graph_recover.node_to_id[node] for node in graph_recover.node_list ] @@ -192,8 +197,8 @@ def test_get_best_model_id(self): """ tuner = NetworkMorphismTuner() - tuner.add_model(0.8, 0) - tuner.add_model(0.9, 1) + tuner.add_model(0.8, 0) + tuner.add_model(0.9, 1) self.assertEqual(tuner.get_best_model_id(), 1) From a852b2801fc5fe083c2064038956cbbcb64ec387 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Fri, 1 Nov 2019 09:44:37 +0800 Subject: [PATCH 10/26] update --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index 4de636f67a..ca98c34d8d 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -122,7 +122,7 @@ def import_data(self, data): a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' """ if not self.values: - logger.info("Search space has not been initialized, skip this data import") + LOGGER.info("Search space has not been initialized, skip this data import") return self.values = self.values[(self.count+1):] From ba51cfbd0df15130d884f43be5d3b287026a67f4 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Fri, 1 Nov 2019 09:48:07 +0800 Subject: [PATCH 11/26] update --- src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py index 07b162d38b..758383f92f 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py @@ -54,12 +54,7 @@ def selection_r(x_bounds, ''' Select using different types. ''' -<<<<<<< HEAD - minimize_starting_points = clusteringmodel_gmm_good.sample( - n_samples=num_starting_points) -======= minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) ->>>>>>> upstream/master outputs = selection(x_bounds, x_types, clusteringmodel_gmm_good, From b3cdbbd0797f196f4ed76f285f59820064aabc1c Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Fri, 1 Nov 2019 09:53:57 +0800 Subject: [PATCH 12/26] update docstring in hyperopt_tuner --- src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index 89c8d662c3..0e250fa8dd 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -422,7 +422,8 @@ def miscs_update_idxs_vals(self, misc_by_id[tid]['vals'][key] = [val] def get_suggestion(self, random_search=False): - """get suggestion from hyperopt + """ + get suggestion from hyperopt Parameters ---------- @@ -473,7 +474,8 @@ def get_suggestion(self, random_search=False): return total_params def import_data(self, data): - """Import additional data for tuning + """ + Import additional data for tuning Parameters ---------- From 86da2d9b76877daf174c52185cb933f4c7021d07 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 10:49:27 +0800 Subject: [PATCH 13/26] update batch_tuner --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index ca98c34d8d..7458a47ad9 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -45,15 +45,15 @@ class BatchTuner(Tuner): Attributes ---------- - count : int - count is the number of parameters that tuner has generated. - values : list - values is to save all the candidates contains in search space. + _count : int + _count is the number of parameters that tuner has generated. + _values : list + _values is to save all the candidates contains in search space. """ def __init__(self): - self.count = -1 - self.values = [] + self._count = -1 + self._values = [] def is_valid(self, search_space): """ @@ -91,7 +91,7 @@ def update_search_space(self, search_space): ---------- search_space : dict """ - self.values = self.is_valid(search_space) + self._values = self.is_valid(search_space) def generate_parameters(self, parameter_id, **kwargs): """Returns a dict of trial (hyper-)parameters, as a serializable object. @@ -105,10 +105,10 @@ def generate_parameters(self, parameter_id, **kwargs): dict A candidate parameter group. """ - self.count += 1 - if self.count > len(self.values) - 1: + self._count += 1 + if self._count > len(self._values) - 1: raise nni.NoMoreTrialError('no more parameters now.') - return self.values[self.count] + return self._values[self._count] def receive_trial_result(self, parameter_id, parameters, value, **kwargs): pass @@ -121,12 +121,12 @@ def import_data(self, data): data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' """ - if not self.values: + if not self._values: LOGGER.info("Search space has not been initialized, skip this data import") return - self.values = self.values[(self.count+1):] - self.count = -1 + self._values = self._values[(self._count+1):] + self._count = -1 _completed_num = 0 for trial_info in data: @@ -141,7 +141,7 @@ def import_data(self, data): LOGGER .info("Useless trial data, value is %s, skip this trial data.", _value) continue _completed_num += 1 - if _params in self.values: - self.values.remove(_params) + if _params in self._values: + self._values.remove(_params) LOGGER .info("Successfully import data to batch tuner, \ total data: %d, imported data: %d.", len(data), _completed_num) From 0ae2aecb893fa1d421d0d1a2dcf8ed4c1888093e Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 10:50:49 +0800 Subject: [PATCH 14/26] delete unused space --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index 7458a47ad9..c05252f584 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -138,7 +138,7 @@ def import_data(self, data): assert "value" in trial_info _value = trial_info['value'] if not _value: - LOGGER .info("Useless trial data, value is %s, skip this trial data.", _value) + LOGGER.info("Useless trial data, value is %s, skip this trial data.", _value) continue _completed_num += 1 if _params in self._values: From 329ef64dbc10163e100fe4fef6e29097762da274 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 10:56:28 +0800 Subject: [PATCH 15/26] update in metis --- .../nni/metis_tuner/Regression_GP/OutlierDetection.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py index c094f82fc3..bb71a51957 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py @@ -16,8 +16,7 @@ # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ OutlierDectection.py """ @@ -33,9 +32,9 @@ def _outlierDetection_threaded(inputs): - ''' + """ Detect the outlier - ''' + """ [samples_idx, samples_x, samples_y_aggregation] = inputs sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n" % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) @@ -59,9 +58,9 @@ def _outlierDetection_threaded(inputs): def outlierDetection_threaded(samples_x, samples_y_aggregation): - ''' + """ Use Multi-thread to detect the outlier - ''' + """ outliers = [] threads_inputs = [[samples_idx, samples_x, samples_y_aggregation] From 61f0844ead662fb062e54a72963dc7d02ba86a50 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 17:02:59 +0800 Subject: [PATCH 16/26] update sdk_reference.rst --- docs/en_US/sdk_reference.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en_US/sdk_reference.rst b/docs/en_US/sdk_reference.rst index 64de1ee45e..a67b25062b 100644 --- a/docs/en_US/sdk_reference.rst +++ b/docs/en_US/sdk_reference.rst @@ -36,6 +36,9 @@ Tuner .. autoclass:: nni.metis_tuner.metis_tuner.MetisTuner :members: +.. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner + :members: + Assessor ------------------------ .. autoclass:: nni.assessor.Assessor From 7aa85172b97a161789de327500c147f2bccbbe11 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 19:15:36 +0800 Subject: [PATCH 17/26] update netowrkmorhism --- src/sdk/pynni/nni/networkmorphism_tuner/graph.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py b/src/sdk/pynni/nni/networkmorphism_tuner/graph.py index 360d61ca11..abf825c48a 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/graph.py @@ -487,9 +487,9 @@ def to_add_skip_model(self, start_id, end_id): # Add the conv layer new_conv_layer = get_conv_class( self.n_dim)( - filters_start, - filters_end, - 1) + filters_start, + filters_end, + 1) skip_output_id = self.add_layer(new_conv_layer, skip_output_id) # Add the add layer. From 7d64494984f19b1ce1f2da40622a0b7910f0aff6 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 19:22:10 +0800 Subject: [PATCH 18/26] update networkmorphsim --- .../networkmorphism_tuner.py | 67 ++++++------------- 1 file changed, 19 insertions(+), 48 deletions(-) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py index 2edeb0bb8e..f7f13d9bda 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py @@ -42,27 +42,27 @@ class NetworkMorphismTuner(Tuner): Attributes ---------- n_classes : int - The class number or output node number (default: {10}) + The class number or output node number (default: ``10``) input_shape : tuple A tuple including: (input_width, input_width, input_channel) t_min : float - The minimum temperature for simulated annealing. (default: {Constant.T_MIN}) + The minimum temperature for simulated annealing. (default: ``Constant.T_MIN``) beta : float - The beta in acquisition function. (default: {Constant.BETA}) + The beta in acquisition function. (default: ``Constant.BETA``) algorithm_name : str - algorithm name used in the network morphism (default: {"Bayesian"}) + algorithm name used in the network morphism (default: ``"Bayesian"``) optimize_mode : str - optimize mode "minimize" or "maximize" (default: {"minimize"}) + optimize mode "minimize" or "maximize" (default: ``"minimize"``) verbose : bool - verbose to print the log (default: {True}) + verbose to print the log (default: ``True``) bo : BayesianOptimizer The optimizer used in networkmorphsim tuner. max_model_size : int - max model size to the graph (default: {Constant.MAX_MODEL_SIZE}) + max model size to the graph (default: ``Constant.MAX_MODEL_SIZE``) default_model_len : int - default model length (default: {Constant.MODEL_LEN}) + default model length (default: ``Constant.MODEL_LEN``) default_model_width : int - default model width (default: {Constant.MODEL_WIDTH}) + default model width (default: ``Constant.MODEL_WIDTH``) search_space : dict """ @@ -84,35 +84,6 @@ def __init__( ): """ initilizer of the NetworkMorphismTuner. - - Parameters - ---------- - task : str - task mode, such as "cv","common" etc. (default: {"cv"}) - input_width : int - input sample shape (default: {32}) - input_channel : int - input sample shape (default: {3}) - n_output_node : int - output node number (default: {10}) - algorithm_name : str - algorithm name used in the network morphism (default: {"Bayesian"}) - optimize_mode : str - optimize mode "minimize" or "maximize" (default: {"minimize"}) - path : str - default mode path to save the model file (default: {"model_path"}) - verbose : bool - verbose to print the log (default: {True}) - beta : float - The beta in acquisition function. (default: {Constant.BETA}) - t_min : float - The minimum temperature for simulated annealing. (default: {Constant.T_MIN}) - max_model_size : int - max model size to the graph (default: {Constant.MAX_MODEL_SIZE}) - default_model_len : int - default model length (default: {Constant.MODEL_LEN}) - default_model_width : int - default model width (default: {Constant.MODEL_WIDTH}) """ if not os.path.exists(path): @@ -237,8 +208,8 @@ def generate(self): """ Generate the next neural architecture. - Returns - ------- + Return + ------ other_info: any object Anything to be saved in the training queue together with the architecture. generated_graph: Graph @@ -281,8 +252,8 @@ def add_model(self, metric_value, model_id): graph : dict model_id : int - Returns - ------- + Return + ------ model : dict """ if self.verbose: @@ -318,8 +289,8 @@ def load_model_by_id(self, model_id): model_id : int model index - Returns - ------- + Return + ------ load_model : Graph the model graph representation """ @@ -334,8 +305,8 @@ def load_best_model(self): """ Get the best model by model id - Returns - ------- + Return + ------ load_model : Graph the model graph representation """ @@ -350,8 +321,8 @@ def get_metric_value_by_id(self, model_id): model_id : int model index - Returns - ------- + Return + ------ float the model metric """ From 76d645a805af689786db605d21380c3fce0b3934 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 19:25:24 +0800 Subject: [PATCH 19/26] update batch_tuner --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index c05252f584..7933f07ce1 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -36,19 +36,17 @@ class BatchTuner class BatchTuner(Tuner): """ BatchTuner is tuner will running all the configure that user want to run batchly. + + Examples + -------- The search space only be accepted like: + ``` { 'combine_params': { '_type': 'choice', '_value': '[{...}, {...}, {...}]', } } - - Attributes - ---------- - _count : int - _count is the number of parameters that tuner has generated. - _values : list - _values is to save all the candidates contains in search space. + ``` """ def __init__(self): @@ -63,8 +61,8 @@ def is_valid(self, search_space): ---------- search_space : dict - Returns - ------- + Return + ------ None or list If valid, return candidate values; else return None. """ @@ -100,8 +98,8 @@ def generate_parameters(self, parameter_id, **kwargs): ---------- parameter_id : int - Returns - ------- + Return + ------ dict A candidate parameter group. """ From a665df865047b698474cd3b143e5d26ed71cc1ec Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 19:34:29 +0800 Subject: [PATCH 20/26] update batch_tuner --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index 7933f07ce1..7e9cdb8582 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -36,7 +36,7 @@ class BatchTuner class BatchTuner(Tuner): """ BatchTuner is tuner will running all the configure that user want to run batchly. - + Examples -------- The search space only be accepted like: From 13a0a0fa6da8356d9348dd6180e70058dd24a606 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 19:37:15 +0800 Subject: [PATCH 21/26] update --- .../pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py index f7f13d9bda..5f3adaf1e6 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py @@ -38,7 +38,7 @@ class NetworkMorphismTuner(Tuner): """ NetworkMorphismTuner is a tuner which using network morphism techniques. - + Attributes ---------- n_classes : int From e7e7fd1c17b32d706fd6d959f16968708ccc734c Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 4 Nov 2019 19:42:47 +0800 Subject: [PATCH 22/26] update metis --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index e57ffcd62e..600cca99b5 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -207,7 +207,7 @@ def update_search_space(self, search_space): "Metis Tuner doesn't support this kind of variable: %s", str(key_type)) raise RuntimeError( - "Metis Tuner doesn't support this kind of variable: %s", + "Metis Tuner doesn't support this kind of variable: %s" % str(key_type)) else: logger.info("The format of search space is not a dict.") From fbceb1426589e52ad26e8c93608c5583a6833fa0 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Thu, 7 Nov 2019 13:44:17 +0800 Subject: [PATCH 23/26] roll back to print --- src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py index bb71a51957..24b2e03027 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py @@ -75,7 +75,7 @@ def outlierDetection_threaded(samples_x, samples_y_aggregation): if threads_result is not None: outliers.append(threads_result) else: - sys.stderr.write("Error: threads_result is None.") + print("Error: threads_result is None.") outliers = outliers if outliers else None return outliers From 27ba59aa09ede536e3d3e8f6ef6f03a2334f83d9 Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Thu, 7 Nov 2019 15:48:18 +0800 Subject: [PATCH 24/26] update Returns --- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 8 +++---- .../nni/evolution_tuner/evolution_tuner.py | 2 +- .../networkmorphism_tuner.py | 24 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index 7e9cdb8582..c223d93552 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -61,8 +61,8 @@ def is_valid(self, search_space): ---------- search_space : dict - Return - ------ + Returns + ------- None or list If valid, return candidate values; else return None. """ @@ -98,8 +98,8 @@ def generate_parameters(self, parameter_id, **kwargs): ---------- parameter_id : int - Return - ------ + Returns + ------- dict A candidate parameter group. """ diff --git a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py index 4da8885be7..1b36a5b1f0 100644 --- a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py +++ b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py @@ -232,7 +232,7 @@ def generate_parameters(self, parameter_id, **kwargs): Returns ------- - config : dict + dict A group of candaidte parameters that evolution tuner generated. """ if not self.population: diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py index 5f3adaf1e6..a5bdec98cb 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py @@ -208,11 +208,11 @@ def generate(self): """ Generate the next neural architecture. - Return - ------ - other_info: any object + Returns + ------- + other_info : any object Anything to be saved in the training queue together with the architecture. - generated_graph: Graph + generated_graph : Graph An instance of Graph. """ generated_graph, new_father_id = self.bo.generate(self.descriptors) @@ -252,8 +252,8 @@ def add_model(self, metric_value, model_id): graph : dict model_id : int - Return - ------ + Returns + ------- model : dict """ if self.verbose: @@ -289,8 +289,8 @@ def load_model_by_id(self, model_id): model_id : int model index - Return - ------ + Returns + ------- load_model : Graph the model graph representation """ @@ -305,8 +305,8 @@ def load_best_model(self): """ Get the best model by model id - Return - ------ + Returns + ------- load_model : Graph the model graph representation """ @@ -321,8 +321,8 @@ def get_metric_value_by_id(self, model_id): model_id : int model index - Return - ------ + Returns + ------- float the model metric """ From 78b61d1c56f750821278a1396781af66ed331ebb Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Thu, 7 Nov 2019 16:02:10 +0800 Subject: [PATCH 25/26] update --- .../pynni/nni/networkmorphism_tuner/layers.py | 273 +++++++++++------- 1 file changed, 164 insertions(+), 109 deletions(-) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py index 1eb86b029c..5658d87a9f 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py @@ -28,8 +28,9 @@ class AvgPool(nn.Module): - '''AvgPool Module. - ''' + """ + AvgPool Module. + """ def __init__(self): super().__init__() @@ -40,8 +41,9 @@ def forward(self, input_tensor): class GlobalAvgPool1d(AvgPool): - '''GlobalAvgPool1d Module. - ''' + """ + GlobalAvgPool1d Module. + """ def forward(self, input_tensor): return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view( @@ -50,8 +52,9 @@ def forward(self, input_tensor): class GlobalAvgPool2d(AvgPool): - '''GlobalAvgPool2d Module. - ''' + """ + GlobalAvgPool2d Module. + """ def forward(self, input_tensor): return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view( @@ -60,8 +63,9 @@ def forward(self, input_tensor): class GlobalAvgPool3d(AvgPool): - '''GlobalAvgPool3d Module. - ''' + """ + GlobalAvgPool3d Module. + """ def forward(self, input_tensor): return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view( @@ -70,8 +74,9 @@ def forward(self, input_tensor): class StubLayer: - '''StubLayer Module. Base Module. - ''' + """ + StubLayer Module. Base Module. + """ def __init__(self, input_node=None, output_node=None): self.input = input_node @@ -79,59 +84,71 @@ def __init__(self, input_node=None, output_node=None): self.weights = None def build(self, shape): - '''build shape. - ''' + """ + build shape. + """ def set_weights(self, weights): - '''set weights. - ''' + """ + set weights. + """ self.weights = weights def import_weights(self, torch_layer): - '''import weights. - ''' + """ + import weights. + """ def import_weights_keras(self, keras_layer): - '''import weights from keras layer. - ''' + """ + import weights from keras layer. + """ def export_weights(self, torch_layer): - '''export weights. - ''' + """ + export weights. + """ def export_weights_keras(self, keras_layer): - '''export weights to keras layer. - ''' + """ + export weights to keras layer. + """ def get_weights(self): - '''get weights. - ''' + """ + get weights. + """ return self.weights def size(self): - '''size(). - ''' + """ + size(). + """ return 0 @property def output_shape(self): - '''output shape. - ''' + """ + output shape. + """ return self.input.shape def to_real_layer(self): - '''to real layer. - ''' + """ + to real layer. + """ def __str__(self): - '''str() function to print. - ''' + """ + str() function to print. + """ return type(self).__name__[4:] class StubWeightBiasLayer(StubLayer): - '''StubWeightBiasLayer Module to set the bias. - ''' + """ + StubWeightBiasLayer Module to set the bias. + """ def import_weights(self, torch_layer): self.set_weights( @@ -151,8 +168,9 @@ def export_weights_keras(self, keras_layer): class StubBatchNormalization(StubWeightBiasLayer): - '''StubBatchNormalization Module. Batch Norm. - ''' + """ + StubBatchNormalization Module. Batch Norm. + """ def __init__(self, num_features, input_node=None, output_node=None): super().__init__(input_node, output_node) @@ -183,32 +201,36 @@ def to_real_layer(self): class StubBatchNormalization1d(StubBatchNormalization): - '''StubBatchNormalization1d Module. - ''' + """ + StubBatchNormalization1d Module. + """ def to_real_layer(self): return torch.nn.BatchNorm1d(self.num_features) class StubBatchNormalization2d(StubBatchNormalization): - '''StubBatchNormalization2d Module. - ''' + """ + StubBatchNormalization2d Module. + """ def to_real_layer(self): return torch.nn.BatchNorm2d(self.num_features) class StubBatchNormalization3d(StubBatchNormalization): - '''StubBatchNormalization3d Module. - ''' + """ + StubBatchNormalization3d Module. + """ def to_real_layer(self): return torch.nn.BatchNorm3d(self.num_features) class StubDense(StubWeightBiasLayer): - '''StubDense Module. Linear. - ''' + """ + StubDense Module. Linear. + """ def __init__(self, input_units, units, input_node=None, output_node=None): super().__init__(input_node, output_node) @@ -235,8 +257,9 @@ def to_real_layer(self): class StubConv(StubWeightBiasLayer): - '''StubConv Module. Conv. - ''' + """ + StubConv Module. Conv. + """ def __init__(self, input_channel, filters, kernel_size, stride=1, input_node=None, output_node=None): @@ -291,8 +314,9 @@ def __str__(self): class StubConv1d(StubConv): - '''StubConv1d Module. - ''' + """ + StubConv1d Module. + """ def to_real_layer(self): return torch.nn.Conv1d( @@ -305,8 +329,9 @@ def to_real_layer(self): class StubConv2d(StubConv): - '''StubConv2d Module. - ''' + """ + StubConv2d Module. + """ def to_real_layer(self): return torch.nn.Conv2d( @@ -319,8 +344,9 @@ def to_real_layer(self): class StubConv3d(StubConv): - '''StubConv3d Module. - ''' + """ + StubConv3d Module. + """ def to_real_layer(self): return torch.nn.Conv3d( @@ -333,8 +359,9 @@ def to_real_layer(self): class StubAggregateLayer(StubLayer): - '''StubAggregateLayer Module. - ''' + """ + StubAggregateLayer Module. + """ def __init__(self, input_nodes=None, output_node=None): if input_nodes is None: @@ -343,8 +370,8 @@ def __init__(self, input_nodes=None, output_node=None): class StubConcatenate(StubAggregateLayer): - '''StubConcatenate Module. - ''' + """StubConcatenate Module. + """ @property def output_shape(self): ret = 0 @@ -358,8 +385,9 @@ def to_real_layer(self): class StubAdd(StubAggregateLayer): - '''StubAdd Module. - ''' + """ + StubAdd Module. + """ @property def output_shape(self): return self.input[0].shape @@ -369,8 +397,9 @@ def to_real_layer(self): class StubFlatten(StubLayer): - '''StubFlatten Module. - ''' + """ + StubFlatten Module. + """ @property def output_shape(self): ret = 1 @@ -383,24 +412,27 @@ def to_real_layer(self): class StubReLU(StubLayer): - '''StubReLU Module. - ''' + """ + StubReLU Module. + """ def to_real_layer(self): return torch.nn.ReLU() class StubSoftmax(StubLayer): - '''StubSoftmax Module. - ''' + """ + StubSoftmax Module. + """ def to_real_layer(self): return torch.nn.LogSoftmax(dim=1) class StubDropout(StubLayer): - '''StubDropout Module. - ''' + """ + StubDropout Module. + """ def __init__(self, rate, input_node=None, output_node=None): super().__init__(input_node, output_node) @@ -412,40 +444,45 @@ def to_real_layer(self): class StubDropout1d(StubDropout): - '''StubDropout1d Module. - ''' + """ + StubDropout1d Module. + """ def to_real_layer(self): return torch.nn.Dropout(self.rate) class StubDropout2d(StubDropout): - '''StubDropout2d Module. - ''' + """ + StubDropout2d Module. + """ def to_real_layer(self): return torch.nn.Dropout2d(self.rate) class StubDropout3d(StubDropout): - '''StubDropout3d Module. - ''' + """ + StubDropout3d Module. + """ def to_real_layer(self): return torch.nn.Dropout3d(self.rate) class StubInput(StubLayer): - '''StubInput Module. - ''' + """ + StubInput Module. + """ def __init__(self, input_node=None, output_node=None): super().__init__(input_node, output_node) class StubPooling(StubLayer): - '''StubPooling Module. - ''' + """ + StubPooling Module. + """ def __init__(self, kernel_size=None, @@ -474,32 +511,36 @@ def to_real_layer(self): class StubPooling1d(StubPooling): - '''StubPooling1d Module. - ''' + """ + StubPooling1d Module. + """ def to_real_layer(self): return torch.nn.MaxPool1d(self.kernel_size, stride=self.stride) class StubPooling2d(StubPooling): - '''StubPooling2d Module. - ''' + """ + StubPooling2d Module. + """ def to_real_layer(self): return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride) class StubPooling3d(StubPooling): - '''StubPooling3d Module. - ''' + """ + StubPooling3d Module. + """ def to_real_layer(self): return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride) class StubGlobalPooling(StubLayer): - '''StubGlobalPooling Module. - ''' + """ + StubGlobalPooling Module. + """ def __init__(self, input_node=None, output_node=None): super().__init__(input_node, output_node) @@ -514,56 +555,63 @@ def to_real_layer(self): class StubGlobalPooling1d(StubGlobalPooling): - '''StubGlobalPooling1d Module. - ''' + """ + StubGlobalPooling1d Module. + """ def to_real_layer(self): return GlobalAvgPool1d() class StubGlobalPooling2d(StubGlobalPooling): - '''StubGlobalPooling2d Module. - ''' + """ + StubGlobalPooling2d Module. + """ def to_real_layer(self): return GlobalAvgPool2d() class StubGlobalPooling3d(StubGlobalPooling): - '''StubGlobalPooling3d Module. - ''' + """ + StubGlobalPooling3d Module. + """ def to_real_layer(self): return GlobalAvgPool3d() class TorchConcatenate(nn.Module): - '''TorchConcatenate Module. - ''' + """ + TorchConcatenate Module. + """ def forward(self, input_list): return torch.cat(input_list, dim=1) class TorchAdd(nn.Module): - '''TorchAdd Module. - ''' + """ + TorchAdd Module. + """ def forward(self, input_list): return input_list[0] + input_list[1] class TorchFlatten(nn.Module): - '''TorchFlatten Module. - ''' + """ + TorchFlatten Module. + """ def forward(self, input_tensor): return input_tensor.view(input_tensor.size(0), -1) def keras_dropout(layer, rate): - '''keras dropout layer. - ''' + """ + Keras dropout layer. + """ from keras import layers @@ -579,8 +627,9 @@ def keras_dropout(layer, rate): def to_real_keras_layer(layer): - ''' real keras layer. - ''' + """ + Real keras layer. + """ from keras import layers if is_layer(layer, "Dense"): @@ -614,10 +663,14 @@ def to_real_keras_layer(layer): def is_layer(layer, layer_type): - '''judge the layer type. - Returns: + """ + Judge the layer type. + + Returns + ------- + bool boolean -- True or False - ''' + """ if layer_type == "Input": return isinstance(layer, StubInput) @@ -647,8 +700,9 @@ def is_layer(layer, layer_type): def layer_description_extractor(layer, node_to_id): - '''get layer description. - ''' + """ + Get layer description. + """ layer_input = layer.input layer_output = layer.output @@ -699,8 +753,8 @@ def layer_description_extractor(layer, node_to_id): def layer_description_builder(layer_information, id_to_node): - '''build layer from description. - ''' + """build layer from description. + """ layer_type = layer_information[0] layer_input_ids = layer_information[1] @@ -737,8 +791,9 @@ def layer_description_builder(layer_information, id_to_node): def layer_width(layer): - '''get layer width. - ''' + """ + Get layer width. + """ if is_layer(layer, "Dense"): return layer.units From 1809c64d6777e5067c9f68fb40c5ebdcda2d3eee Mon Sep 17 00:00:00 2001 From: Xiaohui Xue Date: Mon, 11 Nov 2019 10:24:12 +0800 Subject: [PATCH 26/26] delete white space --- src/sdk/pynni/nni/networkmorphism_tuner/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py index 5658d87a9f..d9c2e5d99e 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py +++ b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py @@ -665,7 +665,7 @@ def to_real_keras_layer(layer): def is_layer(layer, layer_type): """ Judge the layer type. - + Returns ------- bool