From 5cd14211f3497e536ce9d67f28c43cba9ac0ed7e Mon Sep 17 00:00:00 2001 From: quzha Date: Thu, 5 Mar 2020 09:24:23 +0800 Subject: [PATCH 1/3] support dict metric data in assessor --- .../curvefitting_assessor.py | 8 +++-- .../medianstop_assessor.py | 17 +++------- src/sdk/pynni/nni/utils.py | 31 ++++++++++++++++++- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py b/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py index a453bd1ecf..1ed9313438 100644 --- a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py +++ b/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py @@ -4,6 +4,7 @@ import logging import datetime from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history from .model_factory import CurveModel logger = logging.getLogger('curvefitting_Assessor') @@ -91,10 +92,11 @@ def assess_trial(self, trial_job_id, trial_history): Exception unrecognize exception in curvefitting_assessor """ - self.trial_history = trial_history + scalar_trial_history = extract_scalar_history(trial_history) + self.trial_history = scalar_trial_history if not self.set_best_performance: return AssessResult.Good - curr_step = len(trial_history) + curr_step = len(scalar_trial_history) if curr_step < self.start_step: return AssessResult.Good @@ -106,7 +108,7 @@ def assess_trial(self, trial_job_id, trial_history): start_time = datetime.datetime.now() # Predict the final result curvemodel = CurveModel(self.target_pos) - predict_y = curvemodel.predict(trial_history) + predict_y = curvemodel.predict(scalar_trial_history) logger.info('Prediction done. Trial job id = %s. Predict value = %s', trial_job_id, predict_y) if predict_y is None: logger.info('wait for more information to predict precisely') diff --git a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py b/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py index da60cc4f22..89a1215471 100644 --- a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py +++ b/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py @@ -3,6 +3,7 @@ import logging from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history logger = logging.getLogger('medianstop_Assessor') @@ -91,20 +92,12 @@ def assess_trial(self, trial_job_id, trial_history): if curr_step < self._start_step: return AssessResult.Good - try: - num_trial_history = [float(ele) for ele in trial_history] - except (TypeError, ValueError) as error: - logger.warning('incorrect data type or value:') - logger.exception(error) - except Exception as error: - logger.warning('unrecognized exception in medianstop_assessor:') - logger.exception(error) - - self._update_data(trial_job_id, num_trial_history) + scalar_trial_history = extract_scalar_history(trial_history) + self._update_data(trial_job_id, scalar_trial_history) if self._high_better: - best_history = max(trial_history) + best_history = max(scalar_trial_history) else: - best_history = min(trial_history) + best_history = min(scalar_trial_history) avg_array = [] for id_ in self._completed_avg_history: diff --git a/src/sdk/pynni/nni/utils.py b/src/sdk/pynni/nni/utils.py index bb7602a4b1..72b1554309 100644 --- a/src/sdk/pynni/nni/utils.py +++ b/src/sdk/pynni/nni/utils.py @@ -62,6 +62,13 @@ def extract_scalar_reward(value, scalar_key='default'): """ Extract scalar reward from trial result. + Parameters + ---------- + value : int, float, dict + the reported final metric data + scalar_key : str + the key name that indicates the numeric number + Raises ------ RuntimeError @@ -78,6 +85,26 @@ def extract_scalar_reward(value, scalar_key='default'): return reward +def extract_scalar_history(trial_history, scalar_key='default'): + """ + Extract scalar value from a list of intermediate results. + + Parameters + ---------- + trial_history : list + accumulated intermediate results of a trial + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + return [extract_scalar_reward(ele) for ele in trial_history] + + def convert_dict2tuple(value): """ convert dict type to tuple to solve unhashable problem. @@ -90,7 +117,9 @@ def convert_dict2tuple(value): def init_dispatcher_logger(): - """ Initialize dispatcher logging configuration""" + """ + Initialize dispatcher logging configuration + """ logger_file_path = 'dispatcher.log' if dispatcher_env_vars.NNI_LOG_DIRECTORY is not None: logger_file_path = os.path.join(dispatcher_env_vars.NNI_LOG_DIRECTORY, logger_file_path) From 577a40501c5c03101bb6e6ea21f5267e2beec860 Mon Sep 17 00:00:00 2001 From: quzha Date: Thu, 5 Mar 2020 10:48:59 +0800 Subject: [PATCH 2/3] fix bug in includeIntermediateResults is true --- src/sdk/pynni/nni/msg_dispatcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 059ecc2522..d66aca458c 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -234,4 +234,5 @@ def _earlystop_notify_tuner(self, data): if multi_thread_enabled(): self._handle_final_metric_data(data) else: + data['value'] = to_json(data['value']) self.enqueue_command(CommandType.ReportMetricData, data) From 7f48a7a72f262f3ac4fd325b12cb032b74c5a5bf Mon Sep 17 00:00:00 2001 From: quzha Date: Thu, 5 Mar 2020 11:48:09 +0800 Subject: [PATCH 3/3] update --- src/sdk/pynni/nni/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/utils.py b/src/sdk/pynni/nni/utils.py index 72b1554309..22351f717d 100644 --- a/src/sdk/pynni/nni/utils.py +++ b/src/sdk/pynni/nni/utils.py @@ -102,7 +102,7 @@ def extract_scalar_history(trial_history, scalar_key='default'): Incorrect final result: the final result should be float/int, or a dict which has a key named "default" whose value is float/int. """ - return [extract_scalar_reward(ele) for ele in trial_history] + return [extract_scalar_reward(ele, scalar_key) for ele in trial_history] def convert_dict2tuple(value):