From d366a679c690269a37869d8907e03df8f7de3479 Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Sun, 7 Feb 2021 15:25:47 +0800 Subject: [PATCH 1/8] Change command lines as serial. --- .../training_service/remote_machine/extends/windowsCommands.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts index c75653b8be..db31fc4e6e 100644 --- a/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts @@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; + return `powershell -command $setEnv = Start-Job -script {$env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}'};wait-job $setEnv;$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { From 97767f3c7e1ea59d44566f54d3b10b15441dcd53 Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 10:06:56 +0800 Subject: [PATCH 2/8] Rename preCommand to pythonPath --- .../TrainingService/RemoteMachineMode.rst | 41 +------------------ .../TrainingService/RemoteMachineMode.rst | 41 +------------------ .../remote_machine/extends/linuxCommands.ts | 10 ++--- .../remote_machine/extends/windowsCommands.ts | 6 +-- .../remote_machine/osCommands.ts | 2 +- .../remote_machine/remoteMachineData.ts | 2 +- .../remote_machine/shellExecutor.ts | 6 +-- 7 files changed, 15 insertions(+), 93 deletions(-) diff --git a/docs/en_US/TrainingService/RemoteMachineMode.rst b/docs/en_US/TrainingService/RemoteMachineMode.rst index fba51c5abb..aa2052bdb5 100644 --- a/docs/en_US/TrainingService/RemoteMachineMode.rst +++ b/docs/en_US/TrainingService/RemoteMachineMode.rst @@ -176,47 +176,8 @@ Use ``examples/trials/mnist-tfv2`` as the example. Below is content of ``example - ip: ${replace_to_your_remote_machine_ip} username: ${replace_to_your_remote_machine_username} sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath} - # Pre-command will be executed before the remote machine executes other commands. # Below is an example of specifying python environment. - # If you want to execute multiple commands, please use "&&" to connect them. - # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH - -The **preCommand** will be executed before the remote machine executes other commands. So you can configure python environment path like this: - -.. code-block:: yaml - - # Linux remote machine - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH - # Windows remote machine - preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path% - -Or if you want to activate the ``virtualenv`` environment: - -.. code-block:: yaml - - # Linux remote machine - preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # Windows remote machine - preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate - -Or if you want to activate the ``conda`` environment: - -.. code-block:: yaml - - # Linux remote machine - preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - # Windows remote machine - preCommand: call activate ${replace_to_conda_env_name} - -If you want multiple commands to be executed, you can use ``&&`` to connect these commands: - -.. code-block:: yaml - - preCommand: command1 && command2 && command3 - -**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``. + pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} Remote machine supports running experiment in reuse mode. In this mode, NNI will reuse remote machine jobs to run as many as possible trials. It can save time of creating new jobs. User needs to make sure each trial can run independent in the same job, for example, avoid loading checkpoint from previous trials. Follow the setting to enable reuse mode: diff --git a/docs/zh_CN/TrainingService/RemoteMachineMode.rst b/docs/zh_CN/TrainingService/RemoteMachineMode.rst index 0dfb0b02c5..e4243875fc 100644 --- a/docs/zh_CN/TrainingService/RemoteMachineMode.rst +++ b/docs/zh_CN/TrainingService/RemoteMachineMode.rst @@ -176,44 +176,5 @@ Windows - ip: ${replace_to_your_remote_machine_ip} username: ${replace_to_your_remote_machine_username} sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath} - # Pre-command will be executed before the remote machine executes other commands. # Below is an example of specifying python environment. - # If you want to execute multiple commands, please use "&&" to connect them. - # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH - -在远程机器执行其他命令之前，将执行 **预命令**。 因此，可以像这样配置 python 环境路径： - -.. code-block:: yaml - - # Linux remote machine - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH - # Windows remote machine - preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path% - -或者，如果想激活 ``virtualen`` 环境： - -.. code-block:: yaml - - # Linux remote machine - preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # Windows remote machine - preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate - -或者，如果想激活 ``conda`` 环境： - -.. code-block:: yaml - - # Linux remote machine - preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - # Windows remote machine - preCommand: call activate ${replace_to_conda_env_name} - -如果要执行多个命令，可以使用 ``&&`` 连接以下命令： - -.. code-block:: yaml - - preCommand: command1 && command2 && command3 - -**注意**：因为 ``preCommand`` 每次都会在其他命令之前执行，所以强烈建议不要设置 **preCommand** 来对系统进行更改，即 ``mkdir`` or ``touch``. + pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} \ No newline at end of file diff --git a/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts index 6c95c4d0bf..d31bf303a3 100644 --- a/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts @@ -26,7 +26,7 @@ class LinuxCommands extends OsCommands { cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code sh $NNI_SYS_DIR/install_nni.sh cd $NNI_SYS_DIR/code - python3 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ + python3.7 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \ --job_id_file ${jobIdFileName} \ --log_collection '${logCollection}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr @@ -34,7 +34,7 @@ class LinuxCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3.7 -m nni.tools.gpu_tool.gpu_metrics_collector`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { @@ -129,11 +129,11 @@ class LinuxCommands extends OsCommands { return command; } - public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{ - if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){ + public setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined{ + if (command === undefined || command === '' || pythonPath === undefined || pythonPath === ''){ return command; } else { - return `${preCommand} && ${command}`; + return `export PATH=${pythonPath}:$PATH && ${command}`; } } diff --git a/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts index db31fc4e6e..0985d8b847 100644 --- a/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts @@ -123,11 +123,11 @@ class WindowsCommands extends OsCommands { return command; } - public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{ - if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){ + public setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined{ + if (command === undefined || command === '' || pythonPath === undefined || pythonPath === ''){ return command; } else { - return `${preCommand} && set prePath=%path% && ${command}`; + return `set path=${pythonPath};%path% && set prePath=%path% && ${command}`; } } diff --git a/ts/nni_manager/training_service/remote_machine/osCommands.ts b/ts/nni_manager/training_service/remote_machine/osCommands.ts index 769135d9fc..4dda92deb9 100644 --- a/ts/nni_manager/training_service/remote_machine/osCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/osCommands.ts @@ -28,7 +28,7 @@ abstract class OsCommands { public abstract killChildProcesses(pidFileName: string, killSelf: boolean): string; public abstract extractFile(tarFileName: string, targetFolder: string): string; public abstract executeScript(script: string, isFile: boolean): string; - public abstract addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined; + public abstract setPythonPath(pythonPath: string | undefined, command: string | undefined): string | undefined; public abstract fileExistCommand(filePath: string): string | undefined; public joinPath(...paths: string[]): string { diff --git a/ts/nni_manager/training_service/remote_machine/remoteMachineData.ts b/ts/nni_manager/training_service/remote_machine/remoteMachineData.ts index e48d1fbf57..1cf51ea50c 100644 --- a/ts/nni_manager/training_service/remote_machine/remoteMachineData.ts +++ b/ts/nni_manager/training_service/remote_machine/remoteMachineData.ts @@ -23,7 +23,7 @@ export class RemoteMachineMeta { //TODO: initialize varialbe in constructor public occupiedGpuIndexMap?: Map; public readonly useActiveGpu?: boolean = false; - public readonly preCommand?: string; + public readonly pythonPath?: string; } /** diff --git a/ts/nni_manager/training_service/remote_machine/shellExecutor.ts b/ts/nni_manager/training_service/remote_machine/shellExecutor.ts index d262ea500f..2cb279f39e 100644 --- a/ts/nni_manager/training_service/remote_machine/shellExecutor.ts +++ b/ts/nni_manager/training_service/remote_machine/shellExecutor.ts @@ -32,7 +32,7 @@ class ShellExecutor { private tempPath: string = ""; private isWindows: boolean = false; private channelDefaultOutputs: string[] = []; - private preCommand: string | undefined; + private pythonPath: string | undefined; constructor() { this.log = getLogger(); @@ -48,7 +48,7 @@ class ShellExecutor { username: rmMeta.username, tryKeyboard: true, }; - this.preCommand = rmMeta.preCommand; + this.pythonPath = rmMeta.pythonPath; this.name = `${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`; if (rmMeta.passwd !== undefined) { connectConfig.password = rmMeta.passwd; @@ -358,7 +358,7 @@ class ShellExecutor { const commandIndex = randomInt(10000); if(this.osCommands !== undefined){ - command = this.osCommands.addPreCommand(this.preCommand, command); + command = this.osCommands.setPythonPath(this.pythonPath, command); } this.log.debug(`remoteExeCommand(${commandIndex}): [${command}]`); From ddf7aa8b36b7cf13626481d3365e3030f07f393e Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 10:09:27 +0800 Subject: [PATCH 3/8] remove test code --- .../training_service/remote_machine/extends/linuxCommands.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts index d31bf303a3..126d879ff1 100644 --- a/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts @@ -26,7 +26,7 @@ class LinuxCommands extends OsCommands { cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code sh $NNI_SYS_DIR/install_nni.sh cd $NNI_SYS_DIR/code - python3.7 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ + python3 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \ --job_id_file ${jobIdFileName} \ --log_collection '${logCollection}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr @@ -34,7 +34,7 @@ class LinuxCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3.7 -m nni.tools.gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { From 5131aaaa886dca2b8106b28dfe3387b534ada915 Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 10:25:11 +0800 Subject: [PATCH 4/8] revert docs --- .../TrainingService/RemoteMachineMode.rst | 41 ++++++++++++++++++- .../TrainingService/RemoteMachineMode.rst | 41 ++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/docs/en_US/TrainingService/RemoteMachineMode.rst b/docs/en_US/TrainingService/RemoteMachineMode.rst index aa2052bdb5..fba51c5abb 100644 --- a/docs/en_US/TrainingService/RemoteMachineMode.rst +++ b/docs/en_US/TrainingService/RemoteMachineMode.rst @@ -176,8 +176,47 @@ Use ``examples/trials/mnist-tfv2`` as the example. Below is content of ``example - ip: ${replace_to_your_remote_machine_ip} username: ${replace_to_your_remote_machine_username} sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath} + # Pre-command will be executed before the remote machine executes other commands. # Below is an example of specifying python environment. - pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} + # If you want to execute multiple commands, please use "&&" to connect them. + # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate + # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} + preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH + +The **preCommand** will be executed before the remote machine executes other commands. So you can configure python environment path like this: + +.. code-block:: yaml + + # Linux remote machine + preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH + # Windows remote machine + preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path% + +Or if you want to activate the ``virtualenv`` environment: + +.. code-block:: yaml + + # Linux remote machine + preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate + # Windows remote machine + preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate + +Or if you want to activate the ``conda`` environment: + +.. code-block:: yaml + + # Linux remote machine + preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} + # Windows remote machine + preCommand: call activate ${replace_to_conda_env_name} + +If you want multiple commands to be executed, you can use ``&&`` to connect these commands: + +.. code-block:: yaml + + preCommand: command1 && command2 && command3 + +**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``. Remote machine supports running experiment in reuse mode. In this mode, NNI will reuse remote machine jobs to run as many as possible trials. It can save time of creating new jobs. User needs to make sure each trial can run independent in the same job, for example, avoid loading checkpoint from previous trials. Follow the setting to enable reuse mode: diff --git a/docs/zh_CN/TrainingService/RemoteMachineMode.rst b/docs/zh_CN/TrainingService/RemoteMachineMode.rst index e4243875fc..0dfb0b02c5 100644 --- a/docs/zh_CN/TrainingService/RemoteMachineMode.rst +++ b/docs/zh_CN/TrainingService/RemoteMachineMode.rst @@ -176,5 +176,44 @@ Windows - ip: ${replace_to_your_remote_machine_ip} username: ${replace_to_your_remote_machine_username} sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath} + # Pre-command will be executed before the remote machine executes other commands. # Below is an example of specifying python environment. - pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} \ No newline at end of file + # If you want to execute multiple commands, please use "&&" to connect them. + # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate + # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} + preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH + +在远程机器执行其他命令之前，将执行 **预命令**。 因此，可以像这样配置 python 环境路径： + +.. code-block:: yaml + + # Linux remote machine + preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH + # Windows remote machine + preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path% + +或者，如果想激活 ``virtualen`` 环境： + +.. code-block:: yaml + + # Linux remote machine + preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate + # Windows remote machine + preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate + +或者，如果想激活 ``conda`` 环境： + +.. code-block:: yaml + + # Linux remote machine + preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} + # Windows remote machine + preCommand: call activate ${replace_to_conda_env_name} + +如果要执行多个命令，可以使用 ``&&`` 连接以下命令： + +.. code-block:: yaml + + preCommand: command1 && command2 && command3 + +**注意**：因为 ``preCommand`` 每次都会在其他命令之前执行，所以强烈建议不要设置 **preCommand** 来对系统进行更改，即 ``mkdir`` or ``touch``. From 30ed92d0ba8177d65b7e29a71b1f70d9b25a94d8 Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 10:41:58 +0800 Subject: [PATCH 5/8] modify schemas file --- ts/nni_manager/rest_server/restValidationSchemas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/nni_manager/rest_server/restValidationSchemas.ts b/ts/nni_manager/rest_server/restValidationSchemas.ts index e7253beba4..c6b6237f15 100644 --- a/ts/nni_manager/rest_server/restValidationSchemas.ts +++ b/ts/nni_manager/rest_server/restValidationSchemas.ts @@ -18,7 +18,7 @@ export namespace ValidationSchemas { gpuIndices: joi.string(), maxTrialNumPerGpu: joi.number(), useActiveGpu: joi.boolean(), - preCommand: joi.string() + pythonPath: joi.string() })), local_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase gpuIndices: joi.string(), From 5c99d02543b316e999c38acdd27db59585345919 Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 12:16:18 +0800 Subject: [PATCH 6/8] modify config_schema.py for rename --- nni/tools/nnictl/config_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nni/tools/nnictl/config_schema.py b/nni/tools/nnictl/config_schema.py index baa02b26e9..c4ee9948d1 100644 --- a/nni/tools/nnictl/config_schema.py +++ b/nni/tools/nnictl/config_schema.py @@ -425,7 +425,7 @@ def validate(self, data): Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), Optional('useActiveGpu'): setType('useActiveGpu', bool), - Optional('preCommand'): setType('preCommand', str) + Optional('pythonPath'): setType('pythonPath', str) }, { 'ip': setType('ip', str), @@ -435,7 +435,7 @@ def validate(self, data): Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), Optional('useActiveGpu'): setType('useActiveGpu', bool), - Optional('preCommand'): setType('preCommand', str) + Optional('pythonPath'): setType('pythonPath', str) })] } From 1147505db27592ba23bdc2b93cecb947dc749e25 Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 15:09:56 +0800 Subject: [PATCH 7/8] modify eng docs --- .../TrainingService/RemoteMachineMode.rst | 41 +------------------ 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/docs/en_US/TrainingService/RemoteMachineMode.rst b/docs/en_US/TrainingService/RemoteMachineMode.rst index fba51c5abb..aa2052bdb5 100644 --- a/docs/en_US/TrainingService/RemoteMachineMode.rst +++ b/docs/en_US/TrainingService/RemoteMachineMode.rst @@ -176,47 +176,8 @@ Use ``examples/trials/mnist-tfv2`` as the example. Below is content of ``example - ip: ${replace_to_your_remote_machine_ip} username: ${replace_to_your_remote_machine_username} sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath} - # Pre-command will be executed before the remote machine executes other commands. # Below is an example of specifying python environment. - # If you want to execute multiple commands, please use "&&" to connect them. - # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH - -The **preCommand** will be executed before the remote machine executes other commands. So you can configure python environment path like this: - -.. code-block:: yaml - - # Linux remote machine - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH - # Windows remote machine - preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path% - -Or if you want to activate the ``virtualenv`` environment: - -.. code-block:: yaml - - # Linux remote machine - preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # Windows remote machine - preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate - -Or if you want to activate the ``conda`` environment: - -.. code-block:: yaml - - # Linux remote machine - preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - # Windows remote machine - preCommand: call activate ${replace_to_conda_env_name} - -If you want multiple commands to be executed, you can use ``&&`` to connect these commands: - -.. code-block:: yaml - - preCommand: command1 && command2 && command3 - -**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``. + pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} Remote machine supports running experiment in reuse mode. In this mode, NNI will reuse remote machine jobs to run as many as possible trials. It can save time of creating new jobs. User needs to make sure each trial can run independent in the same job, for example, avoid loading checkpoint from previous trials. Follow the setting to enable reuse mode: From 72d38580f399bdbdc46d5afc615e45ec39ea39cc Mon Sep 17 00:00:00 2001 From: Hao Ni Date: Mon, 22 Feb 2021 16:19:57 +0800 Subject: [PATCH 8/8] modify ExperimentConfig.rst --- docs/en_US/Tutorial/ExperimentConfig.rst | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/en_US/Tutorial/ExperimentConfig.rst b/docs/en_US/Tutorial/ExperimentConfig.rst index dc72c27ae5..5da4dfcc4c 100644 --- a/docs/en_US/Tutorial/ExperimentConfig.rst +++ b/docs/en_US/Tutorial/ExperimentConfig.rst @@ -71,7 +71,7 @@ This document describes the rules to write the config file, and provides some ex * `gpuIndices <#gpuindices-3>`__ * `maxTrialNumPerGpu <#maxtrialnumpergpu-1>`__ * `useActiveGpu <#useactivegpu-1>`__ - * `preCommand <#preCommand>`__ + * `pythonPath <#pythonPath>`__ * `kubeflowConfig <#kubeflowconfig>`__ @@ -702,14 +702,12 @@ Optional. Bool. Default: false. Used to specify whether to use a GPU if there is another process. By default, NNI will use the GPU only if there is no other active process in the GPU. If **useActiveGpu** is set to true, NNI will use the GPU regardless of another processes. This field is not applicable for NNI on Windows. -preCommand +pythonPath ^^^^^^^^^^ Optional. String. -Specifies the pre-command that will be executed before the remote machine executes other commands. Users can configure the experimental environment on remote machine by setting **preCommand**. If there are multiple commands need to execute, use ``&&`` to connect them, such as ``preCommand: command1 && command2 && ...``. - -**Note**\ : Because **preCommand** will execute before other commands each time, it is strongly not recommended to set **preCommand** that will make changes to system, i.e. ``mkdir`` or ``touch``. +Users can configure the python path environment on remote machine by setting **pythonPath**. remoteConfig ^^^^^^^^^^^^ @@ -961,12 +959,8 @@ If run trial jobs in remote machine, users could specify the remote machine info username: test sshKeyPath: /nni/sshkey passphrase: qwert - # Pre-command will be executed before the remote machine executes other commands. # Below is an example of specifying python environment. - # If you want to execute multiple commands, please use "&&" to connect them. - # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH + pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} PAI mode ^^^^^^^^