From 67453d107cc655f33c3febd5824b6901a52ae3f0 Mon Sep 17 00:00:00 2001
From: SparkSnail <shinaiyang@pku.edu.cn>
Date: Wed, 31 Oct 2018 11:31:06 +0800
Subject: [PATCH] Show error information and fix paramiko installation (#282)
* fix paramiko install
---
setup.py | 3 +--
tools/bash-completion | 18 +++++++++---------
tools/nnicmd/launcher.py | 39 ++++++++++++++++++++++++++++++---------
tools/nnicmd/ssh_utils.py | 12 +++++++++++-
tools/setup.py | 3 +--
5 files changed, 52 insertions(+), 23 deletions(-)
diff --git a/setup.py b/setup.py
index 1860a58869..ea38f80667 100644
--- a/setup.py
+++ b/setup.py
@@ -62,8 +62,7 @@ def run(self):
'requests',
'scipy',
'schema',
- 'pyhdfs',
- 'paramiko'
+ 'pyhdfs'
],
cmdclass={
diff --git a/tools/bash-completion b/tools/bash-completion
index 11640059ed..408e68e364 100644
--- a/tools/bash-completion
+++ b/tools/bash-completion
@@ -1,19 +1,19 @@
# list of commands/arguments
-__nnictl_cmds="create resume update stop trial experiment config rest log"
-__nnictl_create_cmds="--config --webuiport"
-__nnictl_resume_cmds="--experiment --manager --webuiport"
-__nnictl_update_cmds="searchspace concurrency duration"
+__nnictl_cmds="create resume update stop trial experiment config webui log"
+__nnictl_create_cmds="--config --port"
+__nnictl_resume_cmds="--port"
+__nnictl_update_cmds="searchspace concurrency duration trialnum"
__nnictl_update_searchspace_cmds="--filename"
__nnictl_update_concurrency_cmds="--value"
__nnictl_update_duration_cmds="--value"
+__nnictl_update_trialnum_cmds="--value"
__nnictl_trial_cmds="ls kill"
__nnictl_trial_kill_cmds="--trialid"
-__nnictl_webui_cmds="start stop url"
-__nnictl_webui_start_cmds="--port"
-__nnictl_experiment_cmds="show"
+__nnictl_webui_cmds="url"
+__nnictl_experiment_cmds="show list status"
+__nnictl_experiment_list_cmds="all"
__nnictl_config_cmds="show"
-__nnictl_rest_cmds="check"
-__nnictl_log_cmds="stdout stderr"
+__nnictl_log_cmds="stdout stderr trial"
__nnictl_log_stdout_cmds="--tail --head --path"
__nnictl_log_stderr_cmds="--tail --head --path"
diff --git a/tools/nnicmd/launcher.py b/tools/nnicmd/launcher.py
index 5bbcd0e217..e6a2ebe78e 100644
--- a/tools/nnicmd/launcher.py
+++ b/tools/nnicmd/launcher.py
@@ -23,7 +23,7 @@
import os
import shutil
import string
-from subprocess import Popen, PIPE, call
+from subprocess import Popen, PIPE, call, check_output
import tempfile
from nni_annotation import *
from .launcher_utils import validate_all_content
@@ -36,6 +36,26 @@
import random
import string
+def get_log_path(config_file_name):
+ '''generate stdout and stderr log path'''
+ stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
+ stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+ return stdout_full_path, stderr_full_path
+
+def print_log_content(config_file_name):
+ '''print log information'''
+ stdout_full_path, stderr_full_path = get_log_path(config_file_name)
+ print_normal(' Stdout:')
+ stdout_cmds = ['cat', stdout_full_path]
+ stdout_content = check_output(stdout_cmds)
+ print(stdout_content.decode('utf-8'))
+ print('\n\n')
+ print_normal(' Stderr:')
+ stderr_cmds = ['cat', stderr_full_path]
+ stderr_content = check_output(stderr_cmds)
+ print(stderr_content.decode('utf-8'))
+
+
def start_rest_server(port, platform, mode, config_file_name, experiment_id=None):
'''Run nni manager process'''
nni_config = Config(config_file_name)
@@ -48,8 +68,7 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
cmds = [manager, '--port', str(port), '--mode', platform, '--start_mode', mode]
if mode == 'resume':
cmds += ['--experiment_id', experiment_id]
- stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
- stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+ stdout_full_path, stderr_full_path = get_log_path(config_file_name)
stdout_file = open(stdout_full_path, 'a+')
stderr_file = open(stderr_full_path, 'a+')
time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
@@ -83,7 +102,7 @@ def set_trial_config(experiment_config, port, config_file_name):
return True
else:
print('Error message is {}'.format(response.text))
- stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+ _, stderr_full_path = get_log_path(config_file_name)
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
return False
@@ -102,7 +121,7 @@ def set_remote_config(experiment_config, port, config_file_name):
if not response or not check_response(response):
if response is not None:
err_message = response.text
- stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+ _, stderr_full_path = get_log_path(config_file_name)
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
return False, err_message
@@ -119,7 +138,7 @@ def set_pai_config(experiment_config, port, config_file_name):
if not response or not response.status_code == 200:
if response is not None:
err_message = response.text
- stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+ _, stderr_full_path = get_log_path(config_file_name)
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
return False, err_message
@@ -185,7 +204,7 @@ def set_experiment(experiment_config, mode, port, config_file_name):
if check_response(response):
return response
else:
- stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+ _, stderr_full_path = get_log_path(config_file_name)
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
print_error('Setting experiment error, error message is {}'.format(response.text))
@@ -220,6 +239,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
print_normal('Successfully started Restful server!')
else:
print_error('Restful server start failed!')
+ print_log_content(config_file_name)
try:
cmds = ['pkill', '-P', str(rest_process.pid)]
call(cmds)
@@ -248,7 +268,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
if set_local_config(experiment_config, args.port, config_file_name):
print_normal('Successfully set local config!')
else:
- print_error('Failed!')
+ print_error('Set local config failed!')
try:
cmds = ['pkill', '-P', str(rest_process.pid)]
call(cmds)
@@ -280,7 +300,8 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
experiment_id = json.loads(response.text).get('experiment_id')
nni_config.set_config('experimentId', experiment_id)
else:
- print_error('Failed!')
+ print_error('Start experiment failed!')
+ print_log_content(config_file_name)
try:
cmds = ['pkill', '-P', str(rest_process.pid)]
call(cmds)
diff --git a/tools/nnicmd/ssh_utils.py b/tools/nnicmd/ssh_utils.py
index befd25deb3..fafed9b9e5 100644
--- a/tools/nnicmd/ssh_utils.py
+++ b/tools/nnicmd/ssh_utils.py
@@ -18,9 +18,17 @@
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-import paramiko
import os
from .common_utils import print_error
+from subprocess import call
+
+def check_environment():
+ '''check if paramiko is installed'''
+ try:
+ import paramiko
+ except:
+ cmds = 'python3 -m pip install --user paramiko'
+ call(cmds, shell=True)
def copy_remote_directory_to_local(sftp, remote_path, local_path):
'''copy remote directory to local machine'''
@@ -41,6 +49,8 @@ def copy_remote_directory_to_local(sftp, remote_path, local_path):
def create_ssh_sftp_client(host_ip, port, username, password):
'''create ssh client'''
try:
+ check_environment()
+ import paramiko
conn = paramiko.Transport(host_ip, port)
conn.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(conn)
diff --git a/tools/setup.py b/tools/setup.py
index 5605926b5f..7b368f4267 100644
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -12,8 +12,7 @@
'psutil',
'astor',
'schema',
- 'pyhdfs',
- 'paramiko'
+ 'pyhdfs'
],
author = 'Microsoft NNI Team',