Change command lines as serial. (#3367)

docs/en_US/TrainingService/RemoteMachineMode.rst

@@ -176,47 +176,8 @@ Use ``examples/trials/mnist-tfv2`` as the example. Below is content of ``example
      - ip: ${replace_to_your_remote_machine_ip}
        username: ${replace_to_your_remote_machine_username}
        sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath}
-       # Pre-command will be executed before the remote machine executes other commands.
        # Below is an example of specifying python environment.
-       # If you want to execute multiple commands, please use "&&" to connect them.
-       # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-       # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-       preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
-
-The **preCommand** will be executed before the remote machine executes other commands. So you can configure python environment path like this:
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
-   # Windows remote machine
-   preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path%
-
-Or if you want to activate the ``virtualenv`` environment:
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-   # Windows remote machine
-   preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate
-
-Or if you want to activate the ``conda`` environment:
-
-.. code-block:: yaml
-
-   # Linux remote machine
-   preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-   # Windows remote machine
-   preCommand: call activate ${replace_to_conda_env_name}
-
-If you want multiple commands to be executed, you can use ``&&`` to connect these commands:
-
-.. code-block:: yaml
-
-   preCommand: command1 && command2 && command3
-
-**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``.
+       pythonPath: ${replace_to_python_environment_path_in_your_remote_machine}
 
 Remote machine supports running experiment in reuse mode. In this mode, NNI will reuse remote machine jobs to run as many as possible trials. It can save time of creating new jobs. User needs to make sure each trial can run independent in the same job, for example, avoid loading checkpoint from previous trials.  
 Follow the setting to enable reuse mode:

docs/en_US/Tutorial/ExperimentConfig.rst

@@ -71,7 +71,7 @@ This document describes the rules to write the config file, and provides some ex
       * `gpuIndices <#gpuindices-3>`__
       * `maxTrialNumPerGpu <#maxtrialnumpergpu-1>`__
       * `useActiveGpu <#useactivegpu-1>`__
-      * `preCommand <#preCommand>`__
+      * `pythonPath <#pythonPath>`__
 
     * `kubeflowConfig <#kubeflowconfig>`__
 
@@ -702,14 +702,12 @@ Optional. Bool. Default: false.
 
 Used to specify whether to use a GPU if there is another process. By default, NNI will use the GPU only if there is no other active process in the GPU. If **useActiveGpu** is set to true, NNI will use the GPU regardless of another processes. This field is not applicable for NNI on Windows.
 
-preCommand
+pythonPath
 ^^^^^^^^^^
 
 Optional. String.
 
-Specifies the pre-command that will be executed before the remote machine executes other commands. Users can configure the experimental environment on remote machine by setting **preCommand**. If there are multiple commands need to execute, use ``&&`` to connect them, such as ``preCommand: command1 && command2 && ...``.
-
-**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``.
+Users can configure the python path environment on remote machine by setting **pythonPath**.
 
 remoteConfig
 ^^^^^^^^^^^^
@@ -961,12 +959,8 @@ If run trial jobs in remote machine, users could specify the remote machine info
          username: test
          sshKeyPath: /nni/sshkey
          passphrase: qwert
-         # Pre-command will be executed before the remote machine executes other commands.
          # Below is an example of specifying python environment.
-         # If you want to execute multiple commands, please use "&&" to connect them.
-         # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
-         # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
-         preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
+         pythonPath: ${replace_to_python_environment_path_in_your_remote_machine}
 
 PAI mode
 ^^^^^^^^

nni/tools/nnictl/config_schema.py

@@ -425,7 +425,7 @@ def validate(self, data):
             Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
             Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
             Optional('useActiveGpu'): setType('useActiveGpu', bool),
-            Optional('preCommand'): setType('preCommand', str)
+            Optional('pythonPath'): setType('pythonPath', str)
         },
         {
             'ip': setType('ip', str),
@@ -435,7 +435,7 @@ def validate(self, data):
             Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
             Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
             Optional('useActiveGpu'): setType('useActiveGpu', bool),
-            Optional('preCommand'): setType('preCommand', str)
+            Optional('pythonPath'): setType('pythonPath', str)
         })]
 }
 

ts/nni_manager/rest_server/restValidationSchemas.ts

@@ -18,7 +18,7 @@ export namespace ValidationSchemas {
                 gpuIndices: joi.string(),
                 maxTrialNumPerGpu: joi.number(),
                 useActiveGpu: joi.boolean(),
-                preCommand: joi.string()
+                pythonPath: joi.string()
             })),
             local_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase
                 gpuIndices: joi.string(),

ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts

@@ -129,11 +129,11 @@ class LinuxCommands extends OsCommands {
         return command;
     }
 
-    public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{
-        if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){
+    public setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined{
+        if (command === undefined || command === '' || pythonPath === undefined || pythonPath === ''){
             return command;
         } else {
-            return `${preCommand} && ${command}`;
+            return `export PATH=${pythonPath}:$PATH && ${command}`;
         }
     }
 

ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts

@@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands {
     }
 
     public generateGpuStatsScript(scriptFolder: string): string {
-        return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
+        return `powershell -command $setEnv = Start-Job -script {$env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}'};wait-job $setEnv;$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
     }
 
     public createFolder(folderName: string, sharedFolder: boolean = false): string {
@@ -123,11 +123,11 @@ class WindowsCommands extends OsCommands {
         return command;
     }
 
-    public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{
-        if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){
+    public setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined{
+        if (command === undefined || command === '' || pythonPath === undefined || pythonPath === ''){
             return command;
         } else {
-            return `${preCommand} && set prePath=%path% && ${command}`;
+            return `set path=${pythonPath};%path% && set prePath=%path% && ${command}`;
         }
     }
 

ts/nni_manager/training_service/remote_machine/osCommands.ts

@@ -28,7 +28,7 @@ abstract class OsCommands {
     public abstract killChildProcesses(pidFileName: string, killSelf: boolean): string;
     public abstract extractFile(tarFileName: string, targetFolder: string): string;
     public abstract executeScript(script: string, isFile: boolean): string;
-    public abstract addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined;
+    public abstract setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined;
     public abstract fileExistCommand(filePath: string): string | undefined;
 
     public joinPath(...paths: string[]): string {

ts/nni_manager/training_service/remote_machine/remoteMachineData.ts

@@ -23,7 +23,7 @@ export class RemoteMachineMeta {
     //TODO: initialize varialbe in constructor
     public occupiedGpuIndexMap?: Map<number, number>;
     public readonly useActiveGpu?: boolean = false;
-    public readonly preCommand?: string;
+    public readonly pythonPath?: string;
 }
 
 /**

ts/nni_manager/training_service/remote_machine/shellExecutor.ts

@@ -32,7 +32,7 @@ class ShellExecutor {
     private tempPath: string = "";
     private isWindows: boolean = false;
     private channelDefaultOutputs: string[] = [];
-    private preCommand: string | undefined;
+    private pythonPath: string | undefined;
 
     constructor() {
         this.log = getLogger();
@@ -48,7 +48,7 @@ class ShellExecutor {
             username: rmMeta.username,
             tryKeyboard: true,
         };
-        this.preCommand = rmMeta.preCommand;
+        this.pythonPath = rmMeta.pythonPath;
         this.name = `${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`;
         if (rmMeta.passwd !== undefined) {
             connectConfig.password = rmMeta.passwd;
@@ -358,7 +358,7 @@ class ShellExecutor {
 
         const commandIndex = randomInt(10000);
         if(this.osCommands !== undefined){
-            command = this.osCommands.addPreCommand(this.preCommand, command);
+            command = this.osCommands.setPythonPath(this.pythonPath, command);
         }
         this.log.debug(`remoteExeCommand(${commandIndex}): [${command}]`);