From 80ccfcc7492a3cf2cee4c114c9a4b2503ee3be86 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Tue, 16 Jul 2019 20:24:20 +0800 Subject: [PATCH 1/9] upload code dir seperately --- .../kubeflow/kubeflowTrainingService.ts | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts index 5f94cd5160..df44a79686 100644 --- a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts @@ -43,6 +43,7 @@ import { KubeflowClusterConfig, KubeflowClusterConfigAzure, KubeflowClusterConfi } from './kubeflowConfig'; import { KubeflowJobInfoCollector } from './kubeflowJobInfoCollector'; import { KubeflowJobRestServer } from './kubeflowJobRestServer'; +import { TrialConfig } from 'training_service/common/trialConfig'; // tslint:disable: no-unsafe-any no-any /** @@ -201,6 +202,10 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber throw new Error('Kubeflow Cluster config is not initialized'); } + if (this.kubeflowTrialConfig === undefined) { + throw new Error('Kubeflow Trial config is not initialized'); + } + let trialJobOutputUrl: string = ''; assert(this.kubeflowClusterConfig.storage === undefined @@ -212,13 +217,17 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber throw new Error('azureStorageClient is not initialized'); } try { - //upload local files to azure storage + //upload script files to azure storage await AzureStorageClientUtility.uploadDirectory(this.azureStorageClient, `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, `${trialLocalTempFolder}`); + //upload code files to azure storage + await AzureStorageClientUtility.uploadDirectory(this.azureStorageClient, + `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, + `${this.kubeflowTrialConfig.codeDir}`); - trialJobOutputUrl = `https://${this.azureStorageAccountName}.file.core.windows.net/${this.azureStorageShare}\ - /${path.join('nni', getExperimentId(), trialJobId, 'output')}`; + trialJobOutputUrl = `https://${this.azureStorageAccountName}.file.core.windows.net/${this.azureStorageShare}` + + `/${path.join('nni', getExperimentId(), trialJobId, 'output')}`; } catch (error) { this.log.error(error); @@ -228,9 +237,10 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber const nfsKubeflowClusterConfig: KubeflowClusterConfigNFS = this.kubeflowClusterConfig; // Creat work dir for current trial in NFS directory await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}`); - // Copy code files from local dir to NFS mounted dir + // Copy script files from local dir to NFS mounted dir await cpp.exec(`cp -r ${trialLocalTempFolder}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); - + // Copy codeDir to NFS mounted dir + await cpp.exec(`cp -r ${this.kubeflowTrialConfig.codeDir}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); const nfsConfig: NFSConfig = nfsKubeflowClusterConfig.nfs; trialJobOutputUrl = `nfs://${nfsConfig.server}:${path.join(nfsConfig.path, 'nni', getExperimentId(), trialJobId, 'output')}`; } @@ -255,13 +265,10 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber } //create tmp trial working folder locally. - await cpp.exec(`mkdir -p ${path.dirname(trialLocalTempFolder)}`); - await cpp.exec(`cp -r ${kubeflowTrialConfig.codeDir} ${trialLocalTempFolder}`); + await cpp.exec(`mkdir -p ${trialLocalTempFolder}`); const runScriptContent : string = CONTAINER_INSTALL_NNI_SHELL_FORMAT; // Write NNI installation file to local tmp files await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), runScriptContent, { encoding: 'utf8' }); - // Create tmp trial working folder locally. - await cpp.exec(`mkdir -p ${trialLocalTempFolder}`); // Write worker file content run_worker.sh to local tmp folders if (kubeflowTrialConfig.worker !== undefined) { From c3aec181648ac0e0ccca6901069354ccddc918fc Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Tue, 16 Jul 2019 21:04:48 +0800 Subject: [PATCH 2/9] fix frameworkcontroller --- .../frameworkcontrollerTrainingService.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts index 578130bfad..0aa39d09f3 100644 --- a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts @@ -201,6 +201,10 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple throw new Error('Kubeflow Cluster config is not initialized'); } + if (this.fcTrialConfig === undefined) { + throw new Error('Kubeflow trial config is not initialized'); + } + let trialJobOutputUrl: string = ''; if (this.fcClusterConfig.storageType === 'azureStorage') { @@ -211,9 +215,12 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple //upload local files to azure storage await AzureStorageClientUtility.uploadDirectory( this.azureStorageClient, `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, `${trialLocalTempFolder}`); + //upload code files to azure storage + await AzureStorageClientUtility.uploadDirectory( + this.azureStorageClient, `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, `${this.fcTrialConfig.codeDir}`); - trialJobOutputUrl = `https://${this.azureStorageAccountName}.file.core.windows.net/\ - ${this.azureStorageShare}/${path.join('nni', getExperimentId(), trialJobId, 'output')}`; + trialJobOutputUrl = `https://${this.azureStorageAccountName}.file.core.windows.net/` + + `${this.azureStorageShare}/${path.join('nni', getExperimentId(), trialJobId, 'output')}`; } catch (error) { this.log.error(error); @@ -226,7 +233,8 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}`); // Copy code files from local dir to NFS mounted dir await cpp.exec(`cp -r ${trialLocalTempFolder}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); - + // Copy codeDir to NFS mounted dir + await cpp.exec(`cp -r ${this.fcTrialConfig.codeDir}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); const nfsConfig: NFSConfig = nfsFrameworkControllerClusterConfig.nfs; trialJobOutputUrl = `nfs://${nfsConfig.server}:${path.join(nfsConfig.path, 'nni', getExperimentId(), trialJobId, 'output')}`; } @@ -257,13 +265,12 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple throw new Error('frameworkcontroller trial config is not initialized'); } - await cpp.exec(`mkdir -p ${path.dirname(trialLocalTempFolder)}`); - await cpp.exec(`cp -r ${this.fcTrialConfig.codeDir} ${trialLocalTempFolder}`); + await cpp.exec(`mkdir -p ${trialLocalTempFolder}`); + const installScriptContent : string = CONTAINER_INSTALL_NNI_SHELL_FORMAT; // Write NNI installation file to local tmp files await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), installScriptContent, { encoding: 'utf8' }); // Create tmp trial working folder locally. - await cpp.exec(`mkdir -p ${trialLocalTempFolder}`); for (const taskRole of this.fcTrialConfig.taskRoles) { const runScriptContent: string = From 991544650f94183e693e94c6a7dbb741345b4a05 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Tue, 16 Jul 2019 21:10:57 +0800 Subject: [PATCH 3/9] remove unused code --- .../kubernetes/kubeflow/kubeflowTrainingService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts index df44a79686..1e63cbe93d 100644 --- a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts @@ -43,7 +43,6 @@ import { KubeflowClusterConfig, KubeflowClusterConfigAzure, KubeflowClusterConfi } from './kubeflowConfig'; import { KubeflowJobInfoCollector } from './kubeflowJobInfoCollector'; import { KubeflowJobRestServer } from './kubeflowJobRestServer'; -import { TrialConfig } from 'training_service/common/trialConfig'; // tslint:disable: no-unsafe-any no-any /** From ffed089b16d18152ddf2ca4fc127c891c29c7a75 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Mon, 22 Jul 2019 10:20:44 +0800 Subject: [PATCH 4/9] fix comments --- .../frameworkcontroller/frameworkcontrollerTrainingService.ts | 2 +- .../kubernetes/kubeflow/kubeflowTrainingService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts index 0aa39d09f3..d5b6e282d9 100644 --- a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts @@ -212,7 +212,7 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple throw new Error('azureStorageClient is not initialized'); } try { - //upload local files to azure storage + //upload local files, including scripts for running the trial and configuration (e.g., hyperparameters) for the trial, to azure storage await AzureStorageClientUtility.uploadDirectory( this.azureStorageClient, `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, `${trialLocalTempFolder}`); //upload code files to azure storage diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts index 1e63cbe93d..c64ecdb011 100644 --- a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts @@ -216,7 +216,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber throw new Error('azureStorageClient is not initialized'); } try { - //upload script files to azure storage + //upload local files, including scripts for running the trial and configuration (e.g., hyperparameters) for the trial, to azure storage await AzureStorageClientUtility.uploadDirectory(this.azureStorageClient, `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, `${trialLocalTempFolder}`); From 2c92ad95f0c89be00da697d0c68e4312aec986b3 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Sun, 3 Nov 2019 19:07:18 +0800 Subject: [PATCH 5/9] init --- .../common/experimentStartupInfo.ts | 2 +- .../training_service/common/util.ts | 26 +++++++++---------- .../local/localTrainingService.ts | 16 ++++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/nni_manager/common/experimentStartupInfo.ts b/src/nni_manager/common/experimentStartupInfo.ts index 5675facdde..ba8bd8789b 100644 --- a/src/nni_manager/common/experimentStartupInfo.ts +++ b/src/nni_manager/common/experimentStartupInfo.ts @@ -43,7 +43,7 @@ class ExperimentStartupInfo { this.initialized = true; if (logDir !== undefined && logDir.length > 0) { - this.logDir = path.join(logDir, getExperimentId()); + this.logDir = path.join(path.normalize(logDir), getExperimentId()); } else { this.logDir = path.join(os.homedir(), 'nni', 'experiments', getExperimentId()); } diff --git a/src/nni_manager/training_service/common/util.ts b/src/nni_manager/training_service/common/util.ts index ef05ac57b3..11265f987a 100644 --- a/src/nni_manager/training_service/common/util.ts +++ b/src/nni_manager/training_service/common/util.ts @@ -70,11 +70,11 @@ export async function validateCodeDir(codeDir: string) : Promise { */ export async function execMkdir(directory: string, share: boolean = false): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe New-Item -Path ${directory} -ItemType "directory" -Force`); + await cpp.exec(`powershell.exe New-Item -Path '${directory}' -ItemType "directory" -Force`); } else if (share) { - await cpp.exec(`(umask 0; mkdir -p ${directory})`); + await cpp.exec(`(umask 0; mkdir -p '${directory}')`); } else { - await cpp.exec(`mkdir -p ${directory}`); + await cpp.exec(`mkdir -p '${directory}'`); } return Promise.resolve(); @@ -87,9 +87,9 @@ export async function execMkdir(directory: string, share: boolean = false): Prom */ export async function execCopydir(source: string, destination: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe Copy-Item ${source} -Destination ${destination} -Recurse`); + await cpp.exec(`powershell.exe Copy-Item '${source}' -Destination '${destination}' -Recurse`); } else { - await cpp.exec(`cp -r ${source} ${destination}`); + await cpp.exec(`cp -r '${source}' '${destination}'`); } return Promise.resolve(); @@ -101,9 +101,9 @@ export async function execCopydir(source: string, destination: string): Promise< */ export async function execNewFile(filename: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe New-Item -Path ${filename} -ItemType "file" -Force`); + await cpp.exec(`powershell.exe New-Item -Path '${filename}' -ItemType "file" -Force`); } else { - await cpp.exec(`touch ${filename}`); + await cpp.exec(`touch '${filename}'`); } return Promise.resolve(); @@ -115,9 +115,9 @@ export async function execNewFile(filename: string): Promise { */ export function runScript(filePath: string): cp.ChildProcess { if (process.platform === 'win32') { - return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file ${filePath}`); + return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file '${filePath}'`); } else { - return cp.exec(`bash ${filePath}`); + return cp.exec(`bash '${filePath}'`); } } @@ -128,9 +128,9 @@ export function runScript(filePath: string): cp.ChildProcess { export async function execTail(filePath: string): Promise { let cmdresult: cpp.childProcessPromise.Result; if (process.platform === 'win32') { - cmdresult = await cpp.exec(`powershell.exe Get-Content ${filePath} -Tail 1`); + cmdresult = await cpp.exec(`powershell.exe Get-Content '${filePath}' -Tail 1`); } else { - cmdresult = await cpp.exec(`tail -n 1 ${filePath}`); + cmdresult = await cpp.exec(`tail -n 1 '${filePath}'`); } return Promise.resolve(cmdresult); @@ -142,9 +142,9 @@ export async function execTail(filePath: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe Remove-Item ${directory} -Recurse -Force`); + await cpp.exec(`powershell.exe Remove-Item '${directory}' -Recurse -Force`); } else { - await cpp.exec(`rm -rf ${directory}`); + await cpp.exec(`rm -rf '${directory}'`); } return Promise.resolve(); diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index 2d4d1a1745..2af861e224 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -380,9 +380,9 @@ class LocalTrainingService implements TrainingService { const envVariables: { key: string; value: string }[] = [ { key: 'NNI_PLATFORM', value: 'local' }, { key: 'NNI_EXP_ID', value: this.experimentId }, - { key: 'NNI_SYS_DIR', value: trialJobDetail.workingDirectory }, + { key: 'NNI_SYS_DIR', value: `'${trialJobDetail.workingDirectory}'` }, { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id }, - { key: 'NNI_OUTPUT_DIR', value: trialJobDetail.workingDirectory }, + { key: 'NNI_OUTPUT_DIR', value: `'${trialJobDetail.workingDirectory}'` }, { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() }, { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() } ]; @@ -490,18 +490,18 @@ class LocalTrainingService implements TrainingService { const script: string[] = []; if (process.platform === 'win32') { script.push( - `cmd.exe /c ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`, + `cmd.exe /c ${localTrialConfig.command} 2>'${path.join(workingDirectory, 'stderr')}'`, `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`, `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`, - `Write $LASTEXITCODE " " $NOW_DATE | Out-File ${path.join(workingDirectory, '.nni', 'state')} -NoNewline -encoding utf8`); + `Write $LASTEXITCODE " " $NOW_DATE | Out-File '${path.join(workingDirectory, '.nni', 'state')}' -NoNewline -encoding utf8`); } else { - script.push(`eval ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`); + script.push(`eval ${localTrialConfig.command} 2>'${path.join(workingDirectory, 'stderr')}'`); if (process.platform === 'darwin') { // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x // Considering the worst case, write 999 to avoid negative duration - script.push(`echo $? \`date +%s999\` >${path.join(workingDirectory, '.nni', 'state')}`); + script.push(`echo $? \`date +%s999\` >'${path.join(workingDirectory, '.nni', 'state')}'`); } else { - script.push(`echo $? \`date +%s%3N\` >${path.join(workingDirectory, '.nni', 'state')}`); + script.push(`echo $? \`date +%s%3N\` >'${path.join(workingDirectory, '.nni', 'state')}'`); } } @@ -522,7 +522,7 @@ class LocalTrainingService implements TrainingService { if (process.platform !== 'win32') { runScriptContent.push('#!/bin/bash'); } - runScriptContent.push(`cd ${this.localTrialConfig.codeDir}`); + runScriptContent.push(`cd '${this.localTrialConfig.codeDir}'`); for (const variable of variables) { runScriptContent.push(setEnvironmentVariable(variable)); } From de89a8960e64325bec28099dcca4e31b75a3543e Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Mon, 4 Nov 2019 10:56:35 +0800 Subject: [PATCH 6/9] fix comments --- src/nni_manager/training_service/common/util.ts | 2 +- .../training_service/local/localTrainingService.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nni_manager/training_service/common/util.ts b/src/nni_manager/training_service/common/util.ts index 11265f987a..b0cc46d66b 100644 --- a/src/nni_manager/training_service/common/util.ts +++ b/src/nni_manager/training_service/common/util.ts @@ -173,7 +173,7 @@ export function setEnvironmentVariable(variable: { key: string; value: string }) if (process.platform === 'win32') { return `$env:${variable.key}="${variable.value}"`; } else { - return `export ${variable.key}=${variable.value}`; + return `export ${variable.key}='${variable.value}'`; } } diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index 2af861e224..c5bb320382 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -380,9 +380,9 @@ class LocalTrainingService implements TrainingService { const envVariables: { key: string; value: string }[] = [ { key: 'NNI_PLATFORM', value: 'local' }, { key: 'NNI_EXP_ID', value: this.experimentId }, - { key: 'NNI_SYS_DIR', value: `'${trialJobDetail.workingDirectory}'` }, + { key: 'NNI_SYS_DIR', value: `${trialJobDetail.workingDirectory}` }, { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id }, - { key: 'NNI_OUTPUT_DIR', value: `'${trialJobDetail.workingDirectory}'` }, + { key: 'NNI_OUTPUT_DIR', value: `${trialJobDetail.workingDirectory}` }, { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() }, { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() } ]; From 38372b99f2ae098fac5957dafe2e1bf3ddc4f614 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Mon, 4 Nov 2019 10:59:53 +0800 Subject: [PATCH 7/9] fix windows quota --- src/nni_manager/training_service/common/util.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nni_manager/training_service/common/util.ts b/src/nni_manager/training_service/common/util.ts index b0cc46d66b..294728ee6d 100644 --- a/src/nni_manager/training_service/common/util.ts +++ b/src/nni_manager/training_service/common/util.ts @@ -70,7 +70,7 @@ export async function validateCodeDir(codeDir: string) : Promise { */ export async function execMkdir(directory: string, share: boolean = false): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe New-Item -Path '${directory}' -ItemType "directory" -Force`); + await cpp.exec(`powershell.exe New-Item -Path "${directory}" -ItemType "directory" -Force`); } else if (share) { await cpp.exec(`(umask 0; mkdir -p '${directory}')`); } else { @@ -87,7 +87,7 @@ export async function execMkdir(directory: string, share: boolean = false): Prom */ export async function execCopydir(source: string, destination: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe Copy-Item '${source}' -Destination '${destination}' -Recurse`); + await cpp.exec(`powershell.exe Copy-Item "${source}" -Destination "${destination}" -Recurse`); } else { await cpp.exec(`cp -r '${source}' '${destination}'`); } @@ -101,7 +101,7 @@ export async function execCopydir(source: string, destination: string): Promise< */ export async function execNewFile(filename: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe New-Item -Path '${filename}' -ItemType "file" -Force`); + await cpp.exec(`powershell.exe New-Item -Path "${filename}" -ItemType "file" -Force`); } else { await cpp.exec(`touch '${filename}'`); } @@ -115,7 +115,7 @@ export async function execNewFile(filename: string): Promise { */ export function runScript(filePath: string): cp.ChildProcess { if (process.platform === 'win32') { - return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file '${filePath}'`); + return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file "${filePath}"`); } else { return cp.exec(`bash '${filePath}'`); } @@ -128,7 +128,7 @@ export function runScript(filePath: string): cp.ChildProcess { export async function execTail(filePath: string): Promise { let cmdresult: cpp.childProcessPromise.Result; if (process.platform === 'win32') { - cmdresult = await cpp.exec(`powershell.exe Get-Content '${filePath}' -Tail 1`); + cmdresult = await cpp.exec(`powershell.exe Get-Content "${filePath}" -Tail 1`); } else { cmdresult = await cpp.exec(`tail -n 1 '${filePath}'`); } @@ -142,7 +142,7 @@ export async function execTail(filePath: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`powershell.exe Remove-Item '${directory}' -Recurse -Force`); + await cpp.exec(`powershell.exe Remove-Item "${directory}" -Recurse -Force`); } else { await cpp.exec(`rm -rf '${directory}'`); } From 1c0122987a1a4635fce5abecf253d0f6538f448a Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Mon, 4 Nov 2019 11:36:37 +0800 Subject: [PATCH 8/9] revert change --- .../training_service/local/localTrainingService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index c5bb320382..dd88e9365b 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -380,9 +380,9 @@ class LocalTrainingService implements TrainingService { const envVariables: { key: string; value: string }[] = [ { key: 'NNI_PLATFORM', value: 'local' }, { key: 'NNI_EXP_ID', value: this.experimentId }, - { key: 'NNI_SYS_DIR', value: `${trialJobDetail.workingDirectory}` }, + { key: 'NNI_SYS_DIR', value: trialJobDetail.workingDirectory }, { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id }, - { key: 'NNI_OUTPUT_DIR', value: `${trialJobDetail.workingDirectory}` }, + { key: 'NNI_OUTPUT_DIR', value: trialJobDetail.workingDirectory }, { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() }, { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() } ]; From 852ebddeaefda9ccb9dcc3f6c6883797e4eec180 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Mon, 4 Nov 2019 11:39:22 +0800 Subject: [PATCH 9/9] fix quota --- .../training_service/local/localTrainingService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index dd88e9365b..17cfc1fab9 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -490,12 +490,12 @@ class LocalTrainingService implements TrainingService { const script: string[] = []; if (process.platform === 'win32') { script.push( - `cmd.exe /c ${localTrialConfig.command} 2>'${path.join(workingDirectory, 'stderr')}'`, + `cmd.exe /c ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`, `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`, `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`, - `Write $LASTEXITCODE " " $NOW_DATE | Out-File '${path.join(workingDirectory, '.nni', 'state')}' -NoNewline -encoding utf8`); + `Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`); } else { - script.push(`eval ${localTrialConfig.command} 2>'${path.join(workingDirectory, 'stderr')}'`); + script.push(`eval ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`); if (process.platform === 'darwin') { // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x // Considering the worst case, write 999 to avoid negative duration