From 751445d3c01dbe96955e4bfb11f9e4954c8a8b49 Mon Sep 17 00:00:00 2001 From: Guoxin Date: Mon, 11 Nov 2019 17:05:49 +0800 Subject: [PATCH] docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor (#1692) # docstr/pylint of GP Tuner & CurveFitting Assessor & MedianStop Assessor --- docs/en_US/sdk_reference.rst | 3 + .../curvefitting_assessor.py | 17 +- .../curvefitting_assessor/curvefunctions.py | 104 +++++----- .../curvefitting_assessor/model_factory.py | 28 +-- src/sdk/pynni/nni/gp_tuner/gp_tuner.py | 106 ++++++----- src/sdk/pynni/nni/gp_tuner/target_space.py | 179 +++++++++++++----- src/sdk/pynni/nni/gp_tuner/util.py | 139 +++++++++++--- .../medianstop_assessor.py | 56 +++--- src/sdk/pynni/nni/tuner.py | 3 +- 9 files changed, 409 insertions(+), 226 deletions(-) diff --git a/docs/en_US/sdk_reference.rst b/docs/en_US/sdk_reference.rst index 7bf274996d..de274fabec 100644 --- a/docs/en_US/sdk_reference.rst +++ b/docs/en_US/sdk_reference.rst @@ -39,6 +39,9 @@ Tuner .. autoclass:: nni.batch_tuner.batch_tuner.BatchTuner :members: +.. autoclass:: nni.gp_tuner.gp_tuner.GPTuner + :members: + Assessor ------------------------ .. autoclass:: nni.assessor.Assessor diff --git a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py b/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py index 37e51bccd7..cf9e217099 100644 --- a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py +++ b/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py @@ -29,13 +29,13 @@ class CurvefittingAssessor(Assessor): Parameters ---------- - epoch_num: int + epoch_num : int The total number of epoch - optimize_mode: str + optimize_mode : str optimize mode, 'maximize' or 'minimize' - start_step: int + start_step : int only after receiving start_step number of reported intermediate results - threshold: float + threshold : float The threshold that we decide to early stop the worse performance curve. """ def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1): @@ -70,9 +70,9 @@ def trial_end(self, trial_job_id, success): Parameters ---------- - trial_job_id: int + trial_job_id : int trial job id - success: bool + success : bool True if succssfully finish the experiment, False otherwise """ if success: @@ -90,9 +90,9 @@ def assess_trial(self, trial_job_id, trial_history): Parameters ---------- - trial_job_id: int + trial_job_id : int trial job id - trial_history: list + trial_history : list The history performance matrix of each trial Returns @@ -105,7 +105,6 @@ def assess_trial(self, trial_job_id, trial_history): Exception unrecognize exception in curvefitting_assessor """ - trial_job_id = trial_job_id self.trial_history = trial_history if not self.set_best_performance: return AssessResult.Good diff --git a/src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py b/src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py index e5972ecff3..575aec2a8f 100644 --- a/src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py +++ b/src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py @@ -14,7 +14,9 @@ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - +""" +A family of functions used by CurvefittingAssessor +""" import numpy as np all_models = {} @@ -29,10 +31,10 @@ def vap(x, a, b, c): Parameters ---------- - x: int - a: float - b: float - c: float + x : int + a : float + b : float + c : float Returns ------- @@ -50,10 +52,10 @@ def pow3(x, c, a, alpha): Parameters ---------- - x: int - c: float - a: float - alpha: float + x : int + c : float + a : float + alpha : float Returns ------- @@ -71,9 +73,9 @@ def linear(x, a, b): Parameters ---------- - x: int - a: float - b: float + x : int + a : float + b : float Returns ------- @@ -91,9 +93,9 @@ def logx_linear(x, a, b): Parameters ---------- - x: int - a: float - b: float + x : int + a : float + b : float Returns ------- @@ -112,10 +114,10 @@ def dr_hill_zero_background(x, theta, eta, kappa): Parameters ---------- - x: int - theta: float - eta: float - kappa: float + x : int + theta : float + eta : float + kappa : float Returns ------- @@ -133,10 +135,10 @@ def log_power(x, a, b, c): Parameters ---------- - x: int - a: float - b: float - c: float + x : int + a : float + b : float + c : float Returns ------- @@ -154,11 +156,11 @@ def pow4(x, alpha, a, b, c): Parameters ---------- - x: int - alpha: float - a: float - b: float - c: float + x : int + alpha : float + a : float + b : float + c : float Returns ------- @@ -177,11 +179,11 @@ def mmf(x, alpha, beta, kappa, delta): Parameters ---------- - x: int - alpha: float - beta: float - kappa: float - delta: float + x : int + alpha : float + beta : float + kappa : float + delta : float Returns ------- @@ -199,11 +201,11 @@ def exp4(x, c, a, b, alpha): Parameters ---------- - x: int - c: float - a: float - b: float - alpha: float + x : int + c : float + a : float + b : float + alpha : float Returns ------- @@ -221,9 +223,9 @@ def ilog2(x, c, a): Parameters ---------- - x: int - c: float - a: float + x : int + c : float + a : float Returns ------- @@ -242,11 +244,11 @@ def weibull(x, alpha, beta, kappa, delta): Parameters ---------- - x: int - alpha: float - beta: float - kappa: float - delta: float + x : int + alpha : float + beta : float + kappa : float + delta : float Returns ------- @@ -264,11 +266,11 @@ def janoschek(x, a, beta, k, delta): Parameters ---------- - x: int - a: float - beta: float - k: float - delta: float + x : int + a : float + beta : float + k : float + delta : float Returns ------- diff --git a/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py b/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py index c276928bc4..6df6066dc0 100644 --- a/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py +++ b/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py @@ -40,7 +40,7 @@ class CurveModel: Parameters ---------- - target_pos: int + target_pos : int The point we need to predict """ def __init__(self, target_pos): @@ -120,14 +120,14 @@ def predict_y(self, model, pos): Parameters ---------- - model: string + model : string name of the curve function model - pos: int + pos : int the epoch number of the position you want to predict Returns ------- - int: + int The expected matrix at pos """ if model_para_num[model] == 2: @@ -143,9 +143,9 @@ def f_comb(self, pos, sample): Parameters ---------- - pos: int + pos : int the epoch number of the position you want to predict - sample: list + sample : list sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} Returns @@ -165,7 +165,7 @@ def normalize_weight(self, samples): Parameters ---------- - samples: list + samples : list a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} @@ -187,7 +187,7 @@ def sigma_sq(self, sample): Parameters ---------- - sample: list + sample : list sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} Returns @@ -206,9 +206,9 @@ def normal_distribution(self, pos, sample): Parameters ---------- - pos: int + pos : int the epoch number of the position you want to predict - sample: list + sample : list sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} Returns @@ -225,7 +225,7 @@ def likelihood(self, samples): Parameters ---------- - sample: list + sample : list sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} Returns @@ -244,7 +244,7 @@ def prior(self, samples): Parameters ---------- - samples: list + samples : list a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} @@ -267,7 +267,7 @@ def target_distribution(self, samples): Parameters ---------- - samples: list + samples : list a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} @@ -322,7 +322,7 @@ def predict(self, trial_history): Parameters ---------- - trial_history: list + trial_history : list The history performance matrix of each trial. Returns diff --git a/src/sdk/pynni/nni/gp_tuner/gp_tuner.py b/src/sdk/pynni/nni/gp_tuner/gp_tuner.py index e2f3b8ee54..22122a6cf2 100644 --- a/src/sdk/pynni/nni/gp_tuner/gp_tuner.py +++ b/src/sdk/pynni/nni/gp_tuner/gp_tuner.py @@ -17,9 +17,11 @@ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' -gp_tuner.py -''' +""" +GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + +See :class:`GPTuner` for details. +""" import warnings import logging @@ -38,18 +40,40 @@ class GPTuner(Tuner): - ''' - GPTuner - ''' + """ + GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize', by default 'maximize' + utility : str + utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'. + kappa : float + value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5. + xi : float + used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0. + nu : float + used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5. + alpha : float + Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations. + By default 1e-6. + cold_start_num : int + Number of random exploration to perform before Gaussian Process. By default 10. + selection_num_warm_up : int + Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000 + selection_num_starting_points : int + Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250. + """ def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, selection_num_warm_up=100000, selection_num_starting_points=250): - self.optimize_mode = OptimizeMode(optimize_mode) + self._optimize_mode = OptimizeMode(optimize_mode) # utility function related - self.utility = utility - self.kappa = kappa - self.xi = xi + self._utility = utility + self._kappa = kappa + self._xi = xi # target space self._space = None @@ -72,30 +96,23 @@ def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5 self._selection_num_starting_points = selection_num_starting_points # num of imported data - self.supplement_data_num = 0 + self._supplement_data_num = 0 def update_search_space(self, search_space): - """Update the self.bounds and self.types by the search_space.json + """ + Update the self.bounds and self.types by the search_space.json file. - Parameters - ---------- - search_space : dict + Override of the abstract method in :class:`~nni.tuner.Tuner`. """ self._space = TargetSpace(search_space, self._random_state) def generate_parameters(self, parameter_id, **kwargs): - """Generate next parameter for trial - If the number of trial result is lower than cold start number, - gp will first randomly generate some parameters. - Otherwise, choose the parameters by the Gussian Process Model - - Parameters - ---------- - parameter_id : int - - Returns - ------- - result : dict + """ + Method which provides one set of hyper-parameters. + If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters. + Otherwise, choose the parameters by the Gussian Process Model. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. """ if self._space.len() < self._cold_start_num: results = self._space.random_sample() @@ -107,7 +124,7 @@ def generate_parameters(self, parameter_id, **kwargs): self._gp.fit(self._space.params, self._space.target) util = UtilityFunction( - kind=self.utility, kappa=self.kappa, xi=self.xi) + kind=self._utility, kappa=self._kappa, xi=self._xi) results = acq_max( f_acq=util.utility, @@ -124,17 +141,13 @@ def generate_parameters(self, parameter_id, **kwargs): return results def receive_trial_result(self, parameter_id, parameters, value, **kwargs): - """Tuner receive result from trial. - - Parameters - ---------- - parameter_id : int - parameters : dict - value : dict/float - if value is dict, it should have "default" key. + """ + Method invoked when a trial reports its final result. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. """ value = extract_scalar_reward(value) - if self.optimize_mode == OptimizeMode.Minimize: + if self._optimize_mode == OptimizeMode.Minimize: value = -value logger.info("Received trial result.") @@ -143,26 +156,27 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs): self._space.register(parameters, value) def import_data(self, data): - """Import additional data for tuning - Parameters - ---------- - data: - a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + Import additional data for tuning. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. """ _completed_num = 0 for trial_info in data: - logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + logger.info( + "Importing data, current processing progress %s / %s", _completed_num, len(data)) _completed_num += 1 assert "parameter" in trial_info _params = trial_info["parameter"] assert "value" in trial_info _value = trial_info['value'] if not _value: - logger.info("Useless trial data, value is %s, skip this trial data.", _value) + logger.info( + "Useless trial data, value is %s, skip this trial data.", _value) continue - self.supplement_data_num += 1 + self._supplement_data_num += 1 _parameter_id = '_'.join( - ["ImportData", str(self.supplement_data_num)]) + ["ImportData", str(self._supplement_data_num)]) self.receive_trial_result( parameter_id=_parameter_id, parameters=_params, value=_value) logger.info("Successfully import data to GP tuner.") diff --git a/src/sdk/pynni/nni/gp_tuner/target_space.py b/src/sdk/pynni/nni/gp_tuner/target_space.py index eacf515267..5e26869c54 100644 --- a/src/sdk/pynni/nni/gp_tuner/target_space.py +++ b/src/sdk/pynni/nni/gp_tuner/target_space.py @@ -17,39 +17,51 @@ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' -target_space.py -''' +""" +Tool class to hold the param-space coordinates (X) and target values (Y). +""" + import numpy as np import nni.parameter_expressions as parameter_expressions def _hashable(params): - """ ensure that an point is hashable by a python dict """ + """ + Transform list params to tuple format. Ensure that an point is hashable by a python dict. + + Parameters + ---------- + params : numpy array + array format of parameters + + Returns + ------- + tuple + tuple format of parameters + """ return tuple(map(float, params)) class TargetSpace(): """ Holds the param-space coordinates (X) and target values (Y) + + Parameters + ---------- + pbounds : dict + Dictionary with parameters names and legal values. + + random_state : int, RandomState, or None + optionally specify a seed for a random number generator, by default None. """ def __init__(self, pbounds, random_state=None): - """ - Parameters - ---------- - pbounds : dict - Dictionary with parameters names as keys and a tuple with minimum - and maximum values. - - random_state : int, RandomState, or None - optionally specify a seed for a random number generator - """ - self.random_state = random_state + self._random_state = random_state # Get the name of the parameters self._keys = sorted(pbounds) + # Create an array with parameters bounds self._bounds = np.array( [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] @@ -71,54 +83,100 @@ def __init__(self, pbounds, random_state=None): self._cache = {} def __contains__(self, params): - ''' + """ check if a parameter is already registered - ''' + + Parameters + ---------- + params : numpy array + + Returns + ------- + bool + True if the parameter is already registered, else false + """ return _hashable(params) in self._cache def len(self): - ''' + """ length of registered params and targets - ''' + + Returns + ------- + int + """ assert len(self._params) == len(self._target) return len(self._target) @property def params(self): - ''' - params: numpy array - ''' + """ + registered parameters + + Returns + ------- + numpy array + """ return self._params @property def target(self): - ''' - target: numpy array - ''' + """ + registered target values + + Returns + ------- + numpy array + """ return self._target @property def dim(self): - ''' - dim: int - length of keys - ''' + """ + dimension of parameters + + Returns + ------- + int + """ return len(self._keys) @property def keys(self): - ''' - keys: numpy array - ''' + """ + keys of parameters + + Returns + ------- + numpy array + """ return self._keys @property def bounds(self): - '''bounds''' + """ + bounds of parameters + + Returns + ------- + numpy array + """ return self._bounds def params_to_array(self, params): - ''' dict to array ''' + """ + dict to array + + Parameters + ---------- + params : dict + dict format of parameters + + Returns + ------- + numpy array + array format of parameters + """ try: assert set(params) == set(self.keys) except AssertionError: @@ -129,11 +187,20 @@ def params_to_array(self, params): return np.asarray([params[key] for key in self.keys]) def array_to_params(self, x): - ''' + """ array to dict maintain int type if the paramters is defined as int in search_space.json - ''' + Parameters + ---------- + x : numpy array + array format of parameters + + Returns + ------- + dict + dict format of parameters + """ try: assert len(x) == len(self.keys) except AssertionError: @@ -159,15 +226,15 @@ def register(self, params, target): Parameters ---------- - x : dict + params : dict + parameters - y : float + target : float target function value """ x = self.params_to_array(params) if x in self: - #raise KeyError('Data point {} is not unique'.format(x)) print('Data point {} is not unique'.format(x)) # Insert data into unique dictionary @@ -180,32 +247,43 @@ def random_sample(self): """ Creates a random point within the bounds of the space. + Returns + ------- + numpy array + one groupe of parameter """ params = np.empty(self.dim) for col, _bound in enumerate(self._bounds): if _bound['_type'] == 'choice': params[col] = parameter_expressions.choice( - _bound['_value'], self.random_state) + _bound['_value'], self._random_state) elif _bound['_type'] == 'randint': - params[col] = self.random_state.randint( + params[col] = self._random_state.randint( _bound['_value'][0], _bound['_value'][1], size=1) elif _bound['_type'] == 'uniform': params[col] = parameter_expressions.uniform( - _bound['_value'][0], _bound['_value'][1], self.random_state) + _bound['_value'][0], _bound['_value'][1], self._random_state) elif _bound['_type'] == 'quniform': params[col] = parameter_expressions.quniform( - _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state) + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) elif _bound['_type'] == 'loguniform': params[col] = parameter_expressions.loguniform( - _bound['_value'][0], _bound['_value'][1], self.random_state) + _bound['_value'][0], _bound['_value'][1], self._random_state) elif _bound['_type'] == 'qloguniform': params[col] = parameter_expressions.qloguniform( - _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self.random_state) + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) return params def max(self): - """Get maximum target value found and corresponding parametes.""" + """ + Get maximum target value found and its corresponding parameters. + + Returns + ------- + dict + target value and parameters, empty dict if nothing registered + """ try: res = { 'target': self.target.max(), @@ -218,7 +296,14 @@ def max(self): return res def res(self): - """Get all target values found and corresponding parametes.""" + """ + Get all target values found and corresponding parameters. + + Returns + ------- + list + a list of target values and their corresponding parameters + """ params = [dict(zip(self.keys, p)) for p in self.params] return [ diff --git a/src/sdk/pynni/nni/gp_tuner/util.py b/src/sdk/pynni/nni/gp_tuner/util.py index 9dac7ff499..ed30a22aab 100644 --- a/src/sdk/pynni/nni/gp_tuner/util.py +++ b/src/sdk/pynni/nni/gp_tuner/util.py @@ -17,9 +17,9 @@ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' -gp_tuner.py -''' +""" +utility functions and classes for GPTuner +""" import warnings import numpy as np @@ -28,9 +28,21 @@ def _match_val_type(vals, bounds): - ''' - Update values in the array, to match their corresponding type - ''' + """ + Update values in the array, to match their corresponding type, make sure the value is legal. + + Parameters + ---------- + vals : numpy array + values of parameters + bounds : numpy array + list of dictionary which stores parameters names and legal values. + + Returns + ------- + vals_new : list + The closest legal value to the original value + """ vals_new = [] for i, bound in enumerate(bounds): @@ -52,32 +64,33 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): A function to find the maximum of the acquisition function It uses a combination of random sampling (cheap) and the 'L-BFGS-B' - optimization method. First by sampling `n_warmup` (1e5) points at random, - and then running L-BFGS-B from `n_iter` (250) random starting points. + optimization method. First by sampling ``num_warmup`` points at random, + and then running L-BFGS-B from ``num_starting_points`` random starting points. Parameters ---------- - :param f_acq: + f_acq : UtilityFunction.utility The acquisition function object that return its point-wise value. - :param gp: + gp : GaussianProcessRegressor A gaussian process fitted to the relevant data. - :param y_max: + y_max : float The current maximum known value of the target function. - :param bounds: + bounds : numpy array The variables bounds to limit the search of the acq max. - :param num_warmup: + num_warmup : int number of times to randomly sample the aquisition function - :param num_starting_points: + num_starting_points : int number of times to run scipy.minimize Returns ------- - :return: x_max, The arg max of the acquisition function. + numpy array + The parameter which achieves max of the acquisition function. """ # Warm up with random points @@ -117,36 +130,70 @@ def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): class UtilityFunction(): """ - An object to compute the acquisition functions. + A class to compute different acquisition function values. + + Parameters + ---------- + kind : string + specification of utility function to use + kappa : float + parameter usedd for 'ucb' acquisition function + xi : float + parameter usedd for 'ei' and 'poi' acquisition function """ def __init__(self, kind, kappa, xi): - """ - If UCB is to be used, a constant kappa is needed. - """ - self.kappa = kappa - - self.xi = xi + self._kappa = kappa + self._xi = xi if kind not in ['ucb', 'ei', 'poi']: err = "The utility function " \ "{} has not been implemented, " \ "please choose one of ucb, ei, or poi.".format(kind) raise NotImplementedError(err) - self.kind = kind + self._kind = kind def utility(self, x, gp, y_max): - '''return utility function''' - if self.kind == 'ucb': - return self._ucb(x, gp, self.kappa) - if self.kind == 'ei': - return self._ei(x, gp, y_max, self.xi) - if self.kind == 'poi': - return self._poi(x, gp, y_max, self.xi) + """ + return utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + + Returns + ------- + function + return corresponding function, return None if parameter is illegal + """ + if self._kind == 'ucb': + return self._ucb(x, gp, self._kappa) + if self._kind == 'ei': + return self._ei(x, gp, y_max, self._xi) + if self._kind == 'poi': + return self._poi(x, gp, y_max, self._xi) return None @staticmethod def _ucb(x, gp, kappa): + """ + Upper Confidence Bound (UCB) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + kappa : float + + Returns + ------- + float + """ with warnings.catch_warnings(): warnings.simplefilter("ignore") mean, std = gp.predict(x, return_std=True) @@ -155,6 +202,22 @@ def _ucb(x, gp, kappa): @staticmethod def _ei(x, gp, y_max, xi): + """ + Expected Improvement (EI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ with warnings.catch_warnings(): warnings.simplefilter("ignore") mean, std = gp.predict(x, return_std=True) @@ -164,6 +227,22 @@ def _ei(x, gp, y_max, xi): @staticmethod def _poi(x, gp, y_max, xi): + """ + Possibility Of Improvement (POI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ with warnings.catch_warnings(): warnings.simplefilter("ignore") mean, std = gp.predict(x, return_std=True) diff --git a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py b/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py index a2543dc539..d2fee423c4 100644 --- a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py +++ b/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py @@ -27,21 +27,21 @@ class MedianstopAssessor(Assessor): Parameters ---------- - optimize_mode: str + optimize_mode : str optimize mode, 'maximize' or 'minimize' - start_step: int + start_step : int only after receiving start_step number of reported intermediate results """ def __init__(self, optimize_mode='maximize', start_step=0): - self.start_step = start_step - self.running_history = dict() - self.completed_avg_history = dict() + self._start_step = start_step + self._running_history = dict() + self._completed_avg_history = dict() if optimize_mode == 'maximize': - self.high_better = True + self._high_better = True elif optimize_mode == 'minimize': - self.high_better = False + self._high_better = False else: - self.high_better = True + self._high_better = True logger.warning('unrecognized optimize_mode %s', optimize_mode) def _update_data(self, trial_job_id, trial_history): @@ -49,35 +49,35 @@ def _update_data(self, trial_job_id, trial_history): Parameters ---------- - trial_job_id: int + trial_job_id : int trial job id - trial_history: list + trial_history : list The history performance matrix of each trial """ - if trial_job_id not in self.running_history: - self.running_history[trial_job_id] = [] - self.running_history[trial_job_id].extend(trial_history[len(self.running_history[trial_job_id]):]) + if trial_job_id not in self._running_history: + self._running_history[trial_job_id] = [] + self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):]) def trial_end(self, trial_job_id, success): """trial_end Parameters ---------- - trial_job_id: int + trial_job_id : int trial job id - success: bool + success : bool True if succssfully finish the experiment, False otherwise """ - if trial_job_id in self.running_history: + if trial_job_id in self._running_history: if success: cnt = 0 history_sum = 0 - self.completed_avg_history[trial_job_id] = [] - for each in self.running_history[trial_job_id]: + self._completed_avg_history[trial_job_id] = [] + for each in self._running_history[trial_job_id]: cnt += 1 history_sum += each - self.completed_avg_history[trial_job_id].append(history_sum / cnt) - self.running_history.pop(trial_job_id) + self._completed_avg_history[trial_job_id].append(history_sum / cnt) + self._running_history.pop(trial_job_id) else: logger.warning('trial_end: trial_job_id does not exist in running_history') @@ -86,9 +86,9 @@ def assess_trial(self, trial_job_id, trial_history): Parameters ---------- - trial_job_id: int + trial_job_id : int trial job id - trial_history: list + trial_history : list The history performance matrix of each trial Returns @@ -102,7 +102,7 @@ def assess_trial(self, trial_job_id, trial_history): unrecognize exception in medianstop_assessor """ curr_step = len(trial_history) - if curr_step < self.start_step: + if curr_step < self._start_step: return AssessResult.Good try: @@ -115,18 +115,18 @@ def assess_trial(self, trial_job_id, trial_history): logger.exception(error) self._update_data(trial_job_id, num_trial_history) - if self.high_better: + if self._high_better: best_history = max(trial_history) else: best_history = min(trial_history) avg_array = [] - for id_ in self.completed_avg_history: - if len(self.completed_avg_history[id_]) >= curr_step: - avg_array.append(self.completed_avg_history[id_][curr_step - 1]) + for id_ in self._completed_avg_history: + if len(self._completed_avg_history[id_]) >= curr_step: + avg_array.append(self._completed_avg_history[id_][curr_step - 1]) if avg_array: avg_array.sort() - if self.high_better: + if self._high_better: median = avg_array[(len(avg_array)-1) // 2] return AssessResult.Bad if best_history < median else AssessResult.Good else: diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py index 177232b7ed..a39ed9ff11 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/src/sdk/pynni/nni/tuner.py @@ -79,7 +79,8 @@ class Tuner(Recoverable): :class:`~nni.smac_tuner.smac_tuner.SMACTuner` :class:`~nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner` :class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` - :class:`~nni.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.metis_tuner.metis_tuner.MetisTuner` + :class:`~nni.gp_tuner.gp_tuner.GPTuner` """ def generate_parameters(self, parameter_id, **kwargs):