Show error information and fix paramiko installation (#282)

setup.py

@@ -62,8 +62,7 @@ def run(self):
         'requests',
         'scipy',
         'schema',
-        'pyhdfs',
-        'paramiko'
+        'pyhdfs'
     ],
 
     cmdclass={

tools/bash-completion

@@ -1,19 +1,19 @@
 # list of commands/arguments
-__nnictl_cmds="create resume update stop trial experiment config rest log"
-__nnictl_create_cmds="--config --webuiport"
-__nnictl_resume_cmds="--experiment --manager --webuiport"
-__nnictl_update_cmds="searchspace concurrency duration"
+__nnictl_cmds="create resume update stop trial experiment config webui log"
+__nnictl_create_cmds="--config --port"
+__nnictl_resume_cmds="--port"
+__nnictl_update_cmds="searchspace concurrency duration trialnum"
 __nnictl_update_searchspace_cmds="--filename"
 __nnictl_update_concurrency_cmds="--value"
 __nnictl_update_duration_cmds="--value"
+__nnictl_update_trialnum_cmds="--value"
 __nnictl_trial_cmds="ls kill"
 __nnictl_trial_kill_cmds="--trialid"
-__nnictl_webui_cmds="start stop url"
-__nnictl_webui_start_cmds="--port"
-__nnictl_experiment_cmds="show"
+__nnictl_webui_cmds="url"
+__nnictl_experiment_cmds="show list status"
+__nnictl_experiment_list_cmds="all"
 __nnictl_config_cmds="show"
-__nnictl_rest_cmds="check"
-__nnictl_log_cmds="stdout stderr"
+__nnictl_log_cmds="stdout stderr trial"
 __nnictl_log_stdout_cmds="--tail --head --path"
 __nnictl_log_stderr_cmds="--tail --head --path"
 

tools/nnicmd/launcher.py

@@ -23,7 +23,7 @@
 import os
 import shutil
 import string
-from subprocess import Popen, PIPE, call
+from subprocess import Popen, PIPE, call, check_output
 import tempfile
 from nni_annotation import *
 from .launcher_utils import validate_all_content
@@ -36,6 +36,26 @@
 import random
 import string
 
+def get_log_path(config_file_name):
+    '''generate stdout and stderr log path'''
+    stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
+    stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+    return stdout_full_path, stderr_full_path
+
+def print_log_content(config_file_name):
+    '''print log information'''
+    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
+    print_normal(' Stdout:')
+    stdout_cmds = ['cat', stdout_full_path]
+    stdout_content = check_output(stdout_cmds)
+    print(stdout_content.decode('utf-8'))
+    print('\n\n')
+    print_normal(' Stderr:')
+    stderr_cmds = ['cat', stderr_full_path]
+    stderr_content = check_output(stderr_cmds)
+    print(stderr_content.decode('utf-8'))
+
+
 def start_rest_server(port, platform, mode, config_file_name, experiment_id=None):
     '''Run nni manager process'''
     nni_config = Config(config_file_name)
@@ -48,8 +68,7 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
     cmds = [manager, '--port', str(port), '--mode', platform, '--start_mode', mode]
     if mode == 'resume':
         cmds += ['--experiment_id', experiment_id]
-    stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout')
-    stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+    stdout_full_path, stderr_full_path = get_log_path(config_file_name)
     stdout_file = open(stdout_full_path, 'a+')
     stderr_file = open(stderr_full_path, 'a+')
     time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
@@ -83,7 +102,7 @@ def set_trial_config(experiment_config, port, config_file_name):
         return True
     else:
         print('Error message is {}'.format(response.text))
-        stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+        _, stderr_full_path = get_log_path(config_file_name)
         with open(stderr_full_path, 'a+') as fout:
             fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
         return False
@@ -102,7 +121,7 @@ def set_remote_config(experiment_config, port, config_file_name):
     if not response or not check_response(response):
         if response is not None:
             err_message = response.text
-            stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+            _, stderr_full_path = get_log_path(config_file_name)
             with open(stderr_full_path, 'a+') as fout:
                 fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
         return False, err_message
@@ -119,7 +138,7 @@ def set_pai_config(experiment_config, port, config_file_name):
     if not response or not response.status_code == 200:
         if response is not None:
             err_message = response.text
-            stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+            _, stderr_full_path = get_log_path(config_file_name)
             with open(stderr_full_path, 'a+') as fout:
                 fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
         return False, err_message
@@ -185,7 +204,7 @@ def set_experiment(experiment_config, mode, port, config_file_name):
     if check_response(response):
         return response
     else:
-        stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr')
+        _, stderr_full_path = get_log_path(config_file_name)
         with open(stderr_full_path, 'a+') as fout:
             fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
         print_error('Setting experiment error, error message is {}'.format(response.text))
@@ -220,6 +239,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
         print_normal('Successfully started Restful server!')
     else:
         print_error('Restful server start failed!')
+        print_log_content(config_file_name)
         try:
             cmds = ['pkill', '-P', str(rest_process.pid)]
             call(cmds)
@@ -248,7 +268,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
         if set_local_config(experiment_config, args.port, config_file_name):
             print_normal('Successfully set local config!')
         else:
-            print_error('Failed!')
+            print_error('Set local config failed!')
             try:
                 cmds = ['pkill', '-P', str(rest_process.pid)]
                 call(cmds)
@@ -280,7 +300,8 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
             experiment_id = json.loads(response.text).get('experiment_id')
         nni_config.set_config('experimentId', experiment_id)
     else:
-        print_error('Failed!')
+        print_error('Start experiment failed!')
+        print_log_content(config_file_name)
         try:
             cmds = ['pkill', '-P', str(rest_process.pid)]
             call(cmds)

tools/nnicmd/ssh_utils.py

@@ -18,9 +18,17 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import paramiko
 import os
 from .common_utils import print_error
+from subprocess import call
+
+def check_environment():
+    '''check if paramiko is installed'''
+    try:
+        import paramiko
+    except:
+        cmds = 'python3 -m pip install --user paramiko'
+        call(cmds, shell=True)
 
 def copy_remote_directory_to_local(sftp, remote_path, local_path):
     '''copy remote directory to local machine'''
@@ -41,6 +49,8 @@ def copy_remote_directory_to_local(sftp, remote_path, local_path):
 def create_ssh_sftp_client(host_ip, port, username, password):
     '''create ssh client'''
     try:
+        check_environment()
+        import paramiko
         conn = paramiko.Transport(host_ip, port)
         conn.connect(username=username, password=password)
         sftp = paramiko.SFTPClient.from_transport(conn)

tools/setup.py

@@ -12,8 +12,7 @@
         'psutil',
         'astor',
         'schema',
-        'pyhdfs',
-        'paramiko'
+        'pyhdfs'
     ],
 
     author = 'Microsoft NNI Team',