diff --git a/setup.py b/setup.py index 1860a58869..ea38f80667 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,7 @@ def run(self): 'requests', 'scipy', 'schema', - 'pyhdfs', - 'paramiko' + 'pyhdfs' ], cmdclass={ diff --git a/tools/bash-completion b/tools/bash-completion index 11640059ed..408e68e364 100644 --- a/tools/bash-completion +++ b/tools/bash-completion @@ -1,19 +1,19 @@ # list of commands/arguments -__nnictl_cmds="create resume update stop trial experiment config rest log" -__nnictl_create_cmds="--config --webuiport" -__nnictl_resume_cmds="--experiment --manager --webuiport" -__nnictl_update_cmds="searchspace concurrency duration" +__nnictl_cmds="create resume update stop trial experiment config webui log" +__nnictl_create_cmds="--config --port" +__nnictl_resume_cmds="--port" +__nnictl_update_cmds="searchspace concurrency duration trialnum" __nnictl_update_searchspace_cmds="--filename" __nnictl_update_concurrency_cmds="--value" __nnictl_update_duration_cmds="--value" +__nnictl_update_trialnum_cmds="--value" __nnictl_trial_cmds="ls kill" __nnictl_trial_kill_cmds="--trialid" -__nnictl_webui_cmds="start stop url" -__nnictl_webui_start_cmds="--port" -__nnictl_experiment_cmds="show" +__nnictl_webui_cmds="url" +__nnictl_experiment_cmds="show list status" +__nnictl_experiment_list_cmds="all" __nnictl_config_cmds="show" -__nnictl_rest_cmds="check" -__nnictl_log_cmds="stdout stderr" +__nnictl_log_cmds="stdout stderr trial" __nnictl_log_stdout_cmds="--tail --head --path" __nnictl_log_stderr_cmds="--tail --head --path" diff --git a/tools/nnicmd/launcher.py b/tools/nnicmd/launcher.py index 5bbcd0e217..e6a2ebe78e 100644 --- a/tools/nnicmd/launcher.py +++ b/tools/nnicmd/launcher.py @@ -23,7 +23,7 @@ import os import shutil import string -from subprocess import Popen, PIPE, call +from subprocess import Popen, PIPE, call, check_output import tempfile from nni_annotation import * from .launcher_utils import validate_all_content @@ -36,6 +36,26 @@ import random import string +def get_log_path(config_file_name): + '''generate stdout and stderr log path''' + stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + return stdout_full_path, stderr_full_path + +def print_log_content(config_file_name): + '''print log information''' + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + print_normal(' Stdout:') + stdout_cmds = ['cat', stdout_full_path] + stdout_content = check_output(stdout_cmds) + print(stdout_content.decode('utf-8')) + print('\n\n') + print_normal(' Stderr:') + stderr_cmds = ['cat', stderr_full_path] + stderr_content = check_output(stderr_cmds) + print(stderr_content.decode('utf-8')) + + def start_rest_server(port, platform, mode, config_file_name, experiment_id=None): '''Run nni manager process''' nni_config = Config(config_file_name) @@ -48,8 +68,7 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None cmds = [manager, '--port', str(port), '--mode', platform, '--start_mode', mode] if mode == 'resume': cmds += ['--experiment_id', experiment_id] - stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout') - stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + stdout_full_path, stderr_full_path = get_log_path(config_file_name) stdout_file = open(stdout_full_path, 'a+') stderr_file = open(stderr_full_path, 'a+') time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) @@ -83,7 +102,7 @@ def set_trial_config(experiment_config, port, config_file_name): return True else: print('Error message is {}'.format(response.text)) - stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + _, stderr_full_path = get_log_path(config_file_name) with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) return False @@ -102,7 +121,7 @@ def set_remote_config(experiment_config, port, config_file_name): if not response or not check_response(response): if response is not None: err_message = response.text - stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + _, stderr_full_path = get_log_path(config_file_name) with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) return False, err_message @@ -119,7 +138,7 @@ def set_pai_config(experiment_config, port, config_file_name): if not response or not response.status_code == 200: if response is not None: err_message = response.text - stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + _, stderr_full_path = get_log_path(config_file_name) with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) return False, err_message @@ -185,7 +204,7 @@ def set_experiment(experiment_config, mode, port, config_file_name): if check_response(response): return response else: - stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + _, stderr_full_path = get_log_path(config_file_name) with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) print_error('Setting experiment error, error message is {}'.format(response.text)) @@ -220,6 +239,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen print_normal('Successfully started Restful server!') else: print_error('Restful server start failed!') + print_log_content(config_file_name) try: cmds = ['pkill', '-P', str(rest_process.pid)] call(cmds) @@ -248,7 +268,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen if set_local_config(experiment_config, args.port, config_file_name): print_normal('Successfully set local config!') else: - print_error('Failed!') + print_error('Set local config failed!') try: cmds = ['pkill', '-P', str(rest_process.pid)] call(cmds) @@ -280,7 +300,8 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen experiment_id = json.loads(response.text).get('experiment_id') nni_config.set_config('experimentId', experiment_id) else: - print_error('Failed!') + print_error('Start experiment failed!') + print_log_content(config_file_name) try: cmds = ['pkill', '-P', str(rest_process.pid)] call(cmds) diff --git a/tools/nnicmd/ssh_utils.py b/tools/nnicmd/ssh_utils.py index befd25deb3..fafed9b9e5 100644 --- a/tools/nnicmd/ssh_utils.py +++ b/tools/nnicmd/ssh_utils.py @@ -18,9 +18,17 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import paramiko import os from .common_utils import print_error +from subprocess import call + +def check_environment(): + '''check if paramiko is installed''' + try: + import paramiko + except: + cmds = 'python3 -m pip install --user paramiko' + call(cmds, shell=True) def copy_remote_directory_to_local(sftp, remote_path, local_path): '''copy remote directory to local machine''' @@ -41,6 +49,8 @@ def copy_remote_directory_to_local(sftp, remote_path, local_path): def create_ssh_sftp_client(host_ip, port, username, password): '''create ssh client''' try: + check_environment() + import paramiko conn = paramiko.Transport(host_ip, port) conn.connect(username=username, password=password) sftp = paramiko.SFTPClient.from_transport(conn) diff --git a/tools/setup.py b/tools/setup.py index 5605926b5f..7b368f4267 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -12,8 +12,7 @@ 'psutil', 'astor', 'schema', - 'pyhdfs', - 'paramiko' + 'pyhdfs' ], author = 'Microsoft NNI Team',