Change command lines as serial.

docs/en_US/TrainingService/RemoteMachineMode.rst

@@ -176,47 +176,8 @@ Use ``examples/trials/mnist-tfv2`` as the example. Below is content of ``example
      - ip: ${replace_to_your_remote_machine_ip}
        username: ${replace_to_your_remote_machine_username}
        sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath}
-       # Pre-command will be executed before the remote machine executes other commands.
        # Below is an example of specifying python environment.
-       # If you want to execute multiple commands, please use "&&" to connect them.
-       # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-       # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-       preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
-
-The **preCommand** will be executed before the remote machine executes other commands. So you can configure python environment path like this:
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
-   # Windows remote machine
-   preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path%
-
-Or if you want to activate the ``virtualenv`` environment:
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-   # Windows remote machine
-   preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate
-
-Or if you want to activate the ``conda`` environment:
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-   # Windows remote machine
-   preCommand: call activate ${replace_to_conda_env_name}
-
-If you want multiple commands to be executed, you can use ``&&`` to connect these commands:
-
-.. code-block:: yaml
-
-   preCommand: command1 && command2 && command3
-
-**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``.
+       pythonPath: ${replace_to_python_environment_path_in_your_remote_machine}
 
 Remote machine supports running experiment in reuse mode. In this mode, NNI will reuse remote machine jobs to run as many as possible trials. It can save time of creating new jobs. User needs to make sure each trial can run independent in the same job, for example, avoid loading checkpoint from previous trials.  
 Follow the setting to enable reuse mode:

docs/zh_CN/TrainingService/RemoteMachineMode.rst

@@ -176,44 +176,5 @@ Windows
      - ip: ${replace_to_your_remote_machine_ip}
        username: ${replace_to_your_remote_machine_username}
        sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath}
-       # Pre-command will be executed before the remote machine executes other commands.
        # Below is an example of specifying python environment.
-       # If you want to execute multiple commands, please use "&&" to connect them.
-       # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-       # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-       preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
-
-在远程机器执行其他命令之前，将执行 **预命令**。 因此，可以像这样配置 python 环境路径：
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
-   # Windows remote machine
-   preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path%
-
-或者，如果想激活 ``virtualen`` 环境：
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-   # Windows remote machine
-   preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate
-
-或者，如果想激活 ``conda`` 环境：
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-   # Windows remote machine
-   preCommand: call activate ${replace_to_conda_env_name}
-
-如果要执行多个命令，可以使用 ``&&`` 连接以下命令：
-
-.. code-block:: yaml
-
-   preCommand: command1 && command2 && command3
-
-**注意**：因为 ``preCommand`` 每次都会在其他命令之前执行，所以强烈建议不要设置 **preCommand** 来对系统进行更改，即 ``mkdir`` or ``touch``.
+       pythonPath: ${replace_to_python_environment_path_in_your_remote_machine}

ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts

@@ -129,11 +129,11 @@ class LinuxCommands extends OsCommands {
         return command;
     }
 
-    public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{
-        if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){
+    public setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined{
+        if (command === undefined || command === '' || pythonPath === undefined || pythonPath === ''){
             return command;
         } else {
-            return `${preCommand} && ${command}`;
+            return `export PATH=${pythonPath}:$PATH && ${command}`;
         }
     }
 

ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts

@@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands {
     }
 
     public generateGpuStatsScript(scriptFolder: string): string {
-        return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
+        return `powershell -command $setEnv = Start-Job -script {$env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}'};wait-job $setEnv;$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
     }
 
     public createFolder(folderName: string, sharedFolder: boolean = false): string {
@@ -123,11 +123,11 @@ class WindowsCommands extends OsCommands {
         return command;
     }
 
-    public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{
-        if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){
+    public setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined{
+        if (command === undefined || command === '' || pythonPath === undefined || pythonPath === ''){
             return command;
         } else {
-            return `${preCommand} && set prePath=%path% && ${command}`;
+            return `set path=${pythonPath};%path% && set prePath=%path% && ${command}`;
         }
     }
 

ts/nni_manager/training_service/remote_machine/osCommands.ts

@@ -28,7 +28,7 @@ abstract class OsCommands {
     public abstract killChildProcesses(pidFileName: string, killSelf: boolean): string;
     public abstract extractFile(tarFileName: string, targetFolder: string): string;
     public abstract executeScript(script: string, isFile: boolean): string;
-    public abstract addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined;
+    public abstract setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined;
     public abstract fileExistCommand(filePath: string): string | undefined;
 
     public joinPath(...paths: string[]): string {

ts/nni_manager/training_service/remote_machine/remoteMachineData.ts

@@ -23,7 +23,7 @@ export class RemoteMachineMeta {
     //TODO: initialize varialbe in constructor
     public occupiedGpuIndexMap?: Map<number, number>;
     public readonly useActiveGpu?: boolean = false;
-    public readonly preCommand?: string;
+    public readonly pythonPath?: string;
 }
 
 /**

ts/nni_manager/training_service/remote_machine/shellExecutor.ts

@@ -32,7 +32,7 @@ class ShellExecutor {
     private tempPath: string = "";
     private isWindows: boolean = false;
     private channelDefaultOutputs: string[] = [];
-    private preCommand: string | undefined;
+    private pythonPath: string | undefined;
 
     constructor() {
         this.log = getLogger();
@@ -48,7 +48,7 @@ class ShellExecutor {
             username: rmMeta.username,
             tryKeyboard: true,
         };
-        this.preCommand = rmMeta.preCommand;
+        this.pythonPath = rmMeta.pythonPath;
         this.name = `${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`;
         if (rmMeta.passwd !== undefined) {
             connectConfig.password = rmMeta.passwd;
@@ -358,7 +358,7 @@ class ShellExecutor {
 
         const commandIndex = randomInt(10000);
         if(this.osCommands !== undefined){
-            command = this.osCommands.addPreCommand(this.preCommand, command);
+            command = this.osCommands.setPythonPath(this.pythonPath, command);
         }
         this.log.debug(`remoteExeCommand(${commandIndex}): [${command}]`);