From 5708e05071854bb4aa2096eadc944f5cfcf5e2e5 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 1 Apr 2021 23:20:51 +0800 Subject: [PATCH 1/2] init --- ts/nni_manager/training_service/common/gpuData.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ts/nni_manager/training_service/common/gpuData.ts b/ts/nni_manager/training_service/common/gpuData.ts index 8767baf6e4..c20a692686 100644 --- a/ts/nni_manager/training_service/common/gpuData.ts +++ b/ts/nni_manager/training_service/common/gpuData.ts @@ -79,6 +79,7 @@ export function parseGpuIndices(gpuIndices?: string): Set | undefined { export const GPU_INFO_COLLECTOR_FORMAT_WINDOWS: string = ` $env:METRIC_OUTPUT_DIR="{0}" -$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow +$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow \ +-redirectStandardOutput {0}\\stdout -redirectStandardError {0}\\stderr Write $app.ID | Out-File {1} -NoNewline -encoding utf8 `; From df3daa5381a82cee14b4b86b90395613bd1c0cd3 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 1 Apr 2021 23:21:35 +0800 Subject: [PATCH 2/2] init --- ts/nni_manager/training_service/common/util.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ts/nni_manager/training_service/common/util.ts b/ts/nni_manager/training_service/common/util.ts index 791d7dcebb..c88c9cd957 100644 --- a/ts/nni_manager/training_service/common/util.ts +++ b/ts/nni_manager/training_service/common/util.ts @@ -236,7 +236,8 @@ export function getScriptName(fileNamePrefix: string): string { } export function getGpuMetricsCollectorBashScriptContent(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector \ +1>${scriptFolder}/stdout 2>${scriptFolder}/stderr`; } export function runGpuMetricsCollector(scriptFolder: string): void {