From a2aa7aaa6e11a07995266cb747a0a4af1b539260 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Fri, 3 Jul 2020 16:53:58 +0800 Subject: [PATCH 1/2] fix docker in aml --- deployment/docker/Dockerfile | 12 ++++++++++++ src/nni_manager/config/aml/amlUtil.py | 5 +---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile index 74835358bf..1f3b75b519 100644 --- a/deployment/docker/Dockerfile +++ b/deployment/docker/Dockerfile @@ -29,6 +29,11 @@ RUN DEBIAN_FRONTEND=noninteractive && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# +# generate python script +# +RUN cp /usr/bin/python3 /usr/bin/python + # # update pip # @@ -69,6 +74,13 @@ RUN python3 -m pip --no-cache-dir install pandas==0.23.4 lightgbm==2.2.2 # RUN python3 -m pip --no-cache-dir install nni +# +# install aml package +# +RUN python3 -m pip --no-cache-dir install azureml +RUN python3 -m pip --no-cache-dir install azureml-sdk + + ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:/bin:/sbin WORKDIR /root diff --git a/src/nni_manager/config/aml/amlUtil.py b/src/nni_manager/config/aml/amlUtil.py index 527388ea25..ca2a5e51a0 100644 --- a/src/nni_manager/config/aml/amlUtil.py +++ b/src/nni_manager/config/aml/amlUtil.py @@ -28,10 +28,7 @@ compute_target = ComputeTarget(workspace=ws, name=args.compute_target) experiment = Experiment(ws, args.experiment_name) run_config = RunConfiguration() - dependencies = CondaDependencies() - dependencies.add_pip_package("azureml-sdk") - dependencies.add_pip_package("azureml") - run_config.environment.python.conda_dependencies = dependencies + run_config.environment.python.user_managed_dependencies = True run_config.environment.docker.enabled = True run_config.environment.docker.base_image = args.docker_image run_config.target = compute_target From fc9494b6e3a86513227cc887a6aad8d31d6774f7 Mon Sep 17 00:00:00 2001 From: Shinai Yang Date: Mon, 6 Jul 2020 23:01:34 +0800 Subject: [PATCH 2/2] fix aml message index --- .../training_service/reusable/aml/amlConfig.ts | 1 + .../reusable/channels/amlCommandChannel.ts | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/nni_manager/training_service/reusable/aml/amlConfig.ts b/src/nni_manager/training_service/reusable/aml/amlConfig.ts index 2c101883e1..dd8c2345d4 100644 --- a/src/nni_manager/training_service/reusable/aml/amlConfig.ts +++ b/src/nni_manager/training_service/reusable/aml/amlConfig.ts @@ -36,4 +36,5 @@ export class AMLTrialConfig extends TrialConfig { export class AMLEnvironmentInformation extends EnvironmentInformation { public amlClient?: AMLClient; + public currentMessageIndex: number = -1; } diff --git a/src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts b/src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts index d57befd507..6fbdf40cef 100644 --- a/src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts +++ b/src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts @@ -14,7 +14,6 @@ class AMLRunnerConnection extends RunnerConnection { export class AMLCommandChannel extends CommandChannel { private stopping: boolean = false; - private currentMessageIndex: number = -1; private sendQueues: [EnvironmentInformation, string][] = []; private readonly NNI_METRICS_PATTERN: string = `NNISDK_MEb'(?.*?)'`; @@ -89,7 +88,9 @@ export class AMLCommandChannel extends CommandChannel { const runnerConnections = [...this.runnerConnections.values()] as AMLRunnerConnection[]; for (const runnerConnection of runnerConnections) { // to loop all commands - const amlClient = (runnerConnection.environment as AMLEnvironmentInformation).amlClient; + const amlEnvironmentInformation: AMLEnvironmentInformation = runnerConnection.environment as AMLEnvironmentInformation; + const amlClient = amlEnvironmentInformation.amlClient; + let currentMessageIndex = amlEnvironmentInformation.currentMessageIndex; if (!amlClient) { throw new Error('AML client not initialized!'); } @@ -97,15 +98,16 @@ export class AMLCommandChannel extends CommandChannel { if (command && Object.prototype.hasOwnProperty.call(command, "trial_runner")) { const messages = command['trial_runner']; if (messages) { - if (messages instanceof Object && this.currentMessageIndex < messages.length - 1) { - for (let index = this.currentMessageIndex + 1; index < messages.length; index ++) { + if (messages instanceof Object && currentMessageIndex < messages.length - 1) { + for (let index = currentMessageIndex + 1; index < messages.length; index ++) { this.handleCommand(runnerConnection.environment, messages[index]); } - this.currentMessageIndex = messages.length - 1; - } else if (this.currentMessageIndex === -1){ + currentMessageIndex = messages.length - 1; + } else if (currentMessageIndex === -1){ this.handleCommand(runnerConnection.environment, messages); - this.currentMessageIndex += 1; + currentMessageIndex += 1; } + amlEnvironmentInformation.currentMessageIndex = currentMessageIndex; } } }