From ba5d18c4ba97066d5db5984c8d2a6fdb67255b1b Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 27 Feb 2020 17:52:07 +0800 Subject: [PATCH] Handle exception and kill restserver in nnictl (#2086) --- tools/nni_cmd/launcher.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index 8571cf6498..2af1c609bb 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -525,7 +525,15 @@ def create_experiment(args): nni_config.set_config('experimentConfig', experiment_config) nni_config.set_config('restServerPort', args.port) - launch_experiment(args, experiment_config, 'new', config_file_name) + try: + launch_experiment(args, experiment_config, 'new', config_file_name) + except Exception as exception: + nni_config = Config(config_file_name) + restServerPid = nni_config.get_config('restServerPid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) def manage_stopped_experiment(args, mode): '''view a stopped experiment''' @@ -553,8 +561,16 @@ def manage_stopped_experiment(args, mode): new_config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) new_nni_config = Config(new_config_file_name) new_nni_config.set_config('experimentConfig', experiment_config) - launch_experiment(args, experiment_config, mode, new_config_file_name, experiment_id) new_nni_config.set_config('restServerPort', args.port) + try: + launch_experiment(args, experiment_config, mode, new_config_file_name, experiment_id) + except Exception as exception: + nni_config = Config(new_config_file_name) + restServerPid = nni_config.get_config('restServerPid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) def view_experiment(args): '''view a stopped experiment'''