diff --git a/.gitignore b/.gitignore index 83049a476e..fd71582a60 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,3 @@ venv.bak/ # VSCode .vscode .vs - -# In case you place source code in ~/nni/ -/experiments diff --git a/README.md b/README.md index 912d412314..0fb8d273ae 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ Within the following table, we summarized the current NNI capabilities, we are g ### **Install** -NNI supports and is tested on Ubuntu >= 16.04, macOS >= 10.14.1, and Windows 10 >= 1809. Simply run the following `pip install` in an environment that has `python 64-bit >= 3.5`. +NNI supports and is tested on Ubuntu >= 16.04, macOS >= 10.14.1, and Windows 10 >= 1809. Simply run the following `pip install` in an environment that has `python 64-bit >= 3.6`. Linux or macOS diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile index b5418ec783..b70a48611f 100644 --- a/deployment/docker/Dockerfile +++ b/deployment/docker/Dockerfile @@ -38,7 +38,7 @@ RUN cp /usr/bin/python3 /usr/bin/python # # update pip # -RUN python3 -m pip install --upgrade pip==20.0.2 setuptools==39.1.0 +RUN python3 -m pip install --upgrade pip==20.0.2 setuptools==41.0.0 # numpy 1.14.3 scipy 1.1.0 RUN python3 -m pip --no-cache-dir install \ diff --git a/docs/en_US/Assessor/CustomizeAssessor.md b/docs/en_US/Assessor/CustomizeAssessor.md index 87e05f7fa8..47f3debf1c 100644 --- a/docs/en_US/Assessor/CustomizeAssessor.md +++ b/docs/en_US/Assessor/CustomizeAssessor.md @@ -54,7 +54,7 @@ assessor: Please noted in **2**. The object `trial_history` are exact the object that Trial send to Assessor by using SDK `report_intermediate_result` function. -The working directory of your assessor is `/nni/experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`, +The working directory of your assessor is `/nni-experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`, More detail example you could see: > * [medianstop-assessor](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/medianstop_assessor) diff --git a/docs/en_US/NAS/SPOS.md b/docs/en_US/NAS/SPOS.md index 0d5df10e19..12e2b83be6 100644 --- a/docs/en_US/NAS/SPOS.md +++ b/docs/en_US/NAS/SPOS.md @@ -75,7 +75,7 @@ Then search with evolution tuner. nnictl create --config config_search.yml ``` -The final architecture exported from every epoch of evolution can be found in `checkpoints` under the working directory of your tuner, which, by default, is `$HOME/nni/experiments/your_experiment_id/log`. +The final architecture exported from every epoch of evolution can be found in `checkpoints` under the working directory of your tuner, which, by default, is `$HOME/nni-experiments/your_experiment_id/log`. ### Step 3. Train from Scratch diff --git a/docs/en_US/SupportedFramework_Library.md b/docs/en_US/SupportedFramework_Library.md index 10a124f603..3ff13351b7 100644 --- a/docs/en_US/SupportedFramework_Library.md +++ b/docs/en_US/SupportedFramework_Library.md @@ -1,5 +1,5 @@ # Framework and Library Supports -With the built-in Python API, NNI naturally supports the hyper parameter tuning and neural network search for all the AI frameworks and libraries who support Python models(`version >= 3.5`). NNI had also provided a set of examples and tutorials for some of the popular scenarios to make jump start easier. +With the built-in Python API, NNI naturally supports the hyper parameter tuning and neural network search for all the AI frameworks and libraries who support Python models(`version >= 3.6`). NNI had also provided a set of examples and tutorials for some of the popular scenarios to make jump start easier. ## Supported AI Frameworks diff --git a/docs/en_US/TrialExample/Trials.md b/docs/en_US/TrialExample/Trials.md index 1212fdc97b..2edf5a3f40 100644 --- a/docs/en_US/TrialExample/Trials.md +++ b/docs/en_US/TrialExample/Trials.md @@ -156,19 +156,19 @@ If NNI Annotation is used, the trial's converted code is in another temporary di #!/bin/bash cd /tmp/user_name/nni/annotation/tmpzj0h72x6 #This is the actual directory export NNI_PLATFORM=local -export NNI_SYS_DIR=/home/user_name/nni/experiments/$experiment_id$/trials/$trial_id$ +export NNI_SYS_DIR=/home/user_name/nni-experiments/$experiment_id$/trials/$trial_id$ export NNI_TRIAL_JOB_ID=nrbb2 -export NNI_OUTPUT_DIR=/home/user_name/nni/experiments/$eperiment_id$/trials/$trial_id$ +export NNI_OUTPUT_DIR=/home/user_name/nni-experiments/$eperiment_id$/trials/$trial_id$ export NNI_TRIAL_SEQ_ID=1 export MULTI_PHASE=false export CUDA_VISIBLE_DEVICES= -eval python3 mnist.py 2>/home/user_name/nni/experiments/$experiment_id$/trials/$trial_id$/stderr -echo $? `date +%s%3N` >/home/user_name/nni/experiments/$experiment_id$/trials/$trial_id$/.nni/state +eval python3 mnist.py 2>/home/user_name/nni-experiments/$experiment_id$/trials/$trial_id$/stderr +echo $? `date +%s%3N` >/home/user_name/nni-experiments/$experiment_id$/trials/$trial_id$/.nni/state ``` ### Other Modes -When running trials on other platforms like remote machine or PAI, the environment variable `NNI_OUTPUT_DIR` only refers to the output directory of the trial, while the trial code and `run.sh` might not be there. However, the `trial.log` will be transmitted back to the local machine in the trial's directory, which defaults to `~/nni/experiments/$experiment_id$/trials/$trial_id$/` +When running trials on other platforms like remote machine or PAI, the environment variable `NNI_OUTPUT_DIR` only refers to the output directory of the trial, while the trial code and `run.sh` might not be there. However, the `trial.log` will be transmitted back to the local machine in the trial's directory, which defaults to `~/nni-experiments/$experiment_id$/trials/$trial_id$/` For more information, please refer to [HowToDebug](../Tutorial/HowToDebug.md). diff --git a/docs/en_US/Tuner/CustomizeAdvisor.md b/docs/en_US/Tuner/CustomizeAdvisor.md index 9dd5bc6105..b368f281f5 100644 --- a/docs/en_US/Tuner/CustomizeAdvisor.md +++ b/docs/en_US/Tuner/CustomizeAdvisor.md @@ -33,7 +33,7 @@ advisor: arg1: value1 ``` -**Note that** The working directory of your advisor is `/nni/experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`. +**Note that** The working directory of your advisor is `/nni-experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`. ## Example diff --git a/docs/en_US/Tuner/CustomizeTuner.md b/docs/en_US/Tuner/CustomizeTuner.md index 42209ba7e8..be7e7c2d6d 100644 --- a/docs/en_US/Tuner/CustomizeTuner.md +++ b/docs/en_US/Tuner/CustomizeTuner.md @@ -87,7 +87,7 @@ parameters = {"dropout": 0.3, "learning_rate": 0.4} value = 0.93 ``` -**Note that** The working directory of your tuner is `/nni/experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`, therefore, if you want to access a file (e.g., `data.txt`) in the directory of your own tuner, you cannot use `open('data.txt', 'r')`. Instead, you should use the following: +**Note that** The working directory of your tuner is `/nni-experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`, therefore, if you want to access a file (e.g., `data.txt`) in the directory of your own tuner, you cannot use `open('data.txt', 'r')`. Instead, you should use the following: ```python _pwd = os.path.dirname(__file__) diff --git a/docs/en_US/Tuner/NetworkmorphismTuner.md b/docs/en_US/Tuner/NetworkmorphismTuner.md index 3bc2047fcf..b300ffc37e 100644 --- a/docs/en_US/Tuner/NetworkmorphismTuner.md +++ b/docs/en_US/Tuner/NetworkmorphismTuner.md @@ -59,7 +59,7 @@ If you want to save and load the **best model**, the following methods are recom ```python # 1. Use NNI API ## You can get the best model ID from WebUI -## or `nni/experiments/experiment_id/log/model_path/best_model.txt' +## or `nni-experiments/experiment_id/log/model_path/best_model.txt' ## read the json string from model file and load it with NNI API with open("best-model.json") as json_file: diff --git a/docs/en_US/Tutorial/InstallationLinux.md b/docs/en_US/Tutorial/InstallationLinux.md index 6e11fd7fac..f531987837 100644 --- a/docs/en_US/Tutorial/InstallationLinux.md +++ b/docs/en_US/Tutorial/InstallationLinux.md @@ -6,7 +6,7 @@ Installation on Linux and macOS follow the same instructions, given below. ### Install NNI through pip - Prerequisite: `python 64-bit >= 3.5` + Prerequisite: `python 64-bit >= 3.6` ```bash python3 -m pip install --upgrade nni @@ -16,7 +16,7 @@ Installation on Linux and macOS follow the same instructions, given below. If you are interested in special or the latest code versions, you can install NNI through source code. - Prerequisites: `python 64-bit >=3.5`, `git`, `wget` + Prerequisites: `python 64-bit >=3.6`, `git`, `wget` ```bash git clone -b v1.7 https://github.com/Microsoft/nni.git diff --git a/docs/en_US/Tutorial/InstallationWin.md b/docs/en_US/Tutorial/InstallationWin.md index 6430a6739e..3e8fc1063c 100644 --- a/docs/en_US/Tutorial/InstallationWin.md +++ b/docs/en_US/Tutorial/InstallationWin.md @@ -2,7 +2,7 @@ ## Prerequires -* Python 3.5 (or above) 64-bit. [Anaconda](https://www.anaconda.com/products/individual) or [Miniconda](https://docs.conda.io/en/latest/miniconda.html) is highly recommended to manage multiple Python environments on Windows. +* Python 3.6 (or above) 64-bit. [Anaconda](https://www.anaconda.com/products/individual) or [Miniconda](https://docs.conda.io/en/latest/miniconda.html) is highly recommended to manage multiple Python environments on Windows. * If it's a newly installed Python environment, it needs to install [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) to support build NNI dependencies like `scikit-learn`. diff --git a/docs/en_US/Tutorial/QuickStart.md b/docs/en_US/Tutorial/QuickStart.md index 5b8e2f50be..982d3a5826 100644 --- a/docs/en_US/Tutorial/QuickStart.md +++ b/docs/en_US/Tutorial/QuickStart.md @@ -2,7 +2,7 @@ ## Installation -We currently support Linux, macOS, and Windows. Ubuntu 16.04 or higher, macOS 10.14.1, and Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`. +We currently support Linux, macOS, and Windows. Ubuntu 16.04 or higher, macOS 10.14.1, and Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.6`. ### Linux and macOS diff --git a/docs/en_US/_templates/index.html b/docs/en_US/_templates/index.html index c4f8209309..fd4c650bb7 100644 --- a/docs/en_US/_templates/index.html +++ b/docs/en_US/_templates/index.html @@ -237,7 +237,7 @@

Install

NNI supports and is tested on Ubuntu >= 16.04, macOS >= 10.14.1, and Windows 10 >= 1809. Simply run the following `pip install` - in an environment that has `python 64-bit >= 3.5`. + in an environment that has `python 64-bit >= 3.6`.

Linux or macOS
python3 -m pip install --upgrade nni
diff --git a/examples/tuners/customized_tuner/setup.py b/examples/tuners/customized_tuner/setup.py index 5a831c7688..9dcd16c76e 100644 --- a/examples/tuners/customized_tuner/setup.py +++ b/examples/tuners/customized_tuner/setup.py @@ -8,7 +8,7 @@ version = '0.1', packages = setuptools.find_packages(exclude=['*test*']), - python_requires = '>=3.5', + python_requires = '>=3.6', classifiers = [ 'Programming Language :: Python :: 3', 'License :: OSI Approved :: MIT License', diff --git a/setup.py b/setup.py index fc86bbc954..ac684706e6 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ def read(fname): 'nni_gpu_tool':'tools/nni_gpu_tool' }, package_data = {'nni': ['**/requirements.txt']}, - python_requires = '>=3.5', + python_requires = '>=3.6', install_requires = [ 'astor', 'hyperopt==0.1.2', diff --git a/src/nni_manager/training_service/dlts/dltsTrainingService.ts b/src/nni_manager/training_service/dlts/dltsTrainingService.ts index 30d8fbcf8d..d1887635f9 100644 --- a/src/nni_manager/training_service/dlts/dltsTrainingService.ts +++ b/src/nni_manager/training_service/dlts/dltsTrainingService.ts @@ -266,7 +266,7 @@ class DLTSTrainingService implements TrainingService { public async submitTrialJob(form: TrialJobApplicationForm): Promise { const trialJobId: string = uniqueString(5); const trialWorkingFolder: string = path.join( - '/nni/experiments', getExperimentId(), + '/nni-experiments', getExperimentId(), '/trials/', trialJobId); const trialJobDetail = new DLTSTrialJobDetail( trialJobId, // id diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts index 56756b708d..2c840100e5 100644 --- a/src/nni_manager/training_service/pai/paiTrainingService.ts +++ b/src/nni_manager/training_service/pai/paiTrainingService.ts @@ -54,7 +54,7 @@ abstract class PAITrainingService implements TrainingService { this.metricsEmitter = new EventEmitter(); this.trialJobsMap = new Map(); this.jobQueue = []; - this.expRootDir = path.join('/nni', 'experiments', getExperimentId()); + this.expRootDir = path.join('/nni-experiments', getExperimentId()); this.experimentId = getExperimentId(); this.paiJobCollector = new PAIJobInfoCollector(this.trialJobsMap); this.paiTokenUpdateInterval = 7200000; //2hours diff --git a/src/nni_manager/training_service/remote_machine/shellExecutor.ts b/src/nni_manager/training_service/remote_machine/shellExecutor.ts index 7f304a1381..5f3a0bc78b 100644 --- a/src/nni_manager/training_service/remote_machine/shellExecutor.ts +++ b/src/nni_manager/training_service/remote_machine/shellExecutor.ts @@ -175,7 +175,7 @@ class ShellExecutor { } public getRemoteExperimentRootDir(experimentId: string): string { - return this.joinPath(this.tempPath, 'nni', 'experiments', experimentId); + return this.joinPath(this.tempPath, 'nni-experiments', experimentId); } public joinPath(...paths: string[]): string { diff --git a/src/sdk/pycli/setup.py b/src/sdk/pycli/setup.py index 15fcee5d0d..4d6492ed67 100644 --- a/src/sdk/pycli/setup.py +++ b/src/sdk/pycli/setup.py @@ -8,7 +8,7 @@ version='999.0.0-developing', packages=setuptools.find_packages(), - python_requires='>=3.5', + python_requires='>=3.6', install_requires=[ 'requests' ], diff --git a/src/sdk/pynni/setup.py b/src/sdk/pynni/setup.py index 4e006a9014..ba6d0d74e8 100644 --- a/src/sdk/pynni/setup.py +++ b/src/sdk/pynni/setup.py @@ -12,7 +12,7 @@ def read(fname): version = '999.0.0-developing', packages = setuptools.find_packages(exclude=['tests']), - python_requires = '>=3.5', + python_requires = '>=3.6', install_requires = [ 'hyperopt==0.1.2', 'json_tricks', diff --git a/src/webui/mock/dict-metrics-100.json b/src/webui/mock/dict-metrics-100.json index 4c6628897d..6179b126ee 100644 --- a/src/webui/mock/dict-metrics-100.json +++ b/src/webui/mock/dict-metrics-100.json @@ -14,7 +14,7 @@ "trainingServicePlatform": "local", "tuner": { "builtinTunerName": "Random", - "checkpointDir": "/home/user/nni/experiments/TN5K0Qju/checkpoint" + "checkpointDir": "/home/user/nni-experiments/TN5K0Qju/checkpoint" }, "versionCheck": true, "clusterMetaData": [ diff --git a/src/webui/mock/mnist-tfv1-nested.json b/src/webui/mock/mnist-tfv1-nested.json index 3028d7f860..f86de77c14 100644 --- a/src/webui/mock/mnist-tfv1-nested.json +++ b/src/webui/mock/mnist-tfv1-nested.json @@ -7,7 +7,7 @@ "id": "AKVcExIf", "revision": 29, "execDuration": 172, - "logDir": "/home/v-yugzh/nni/experiments/AKVcExIf", + "logDir": "/home/v-yugzh/nni-experiments/AKVcExIf", "nextSequenceId": 10, "params": { "authorName": "default", diff --git a/src/webui/mock/mnist-tfv1-running.json b/src/webui/mock/mnist-tfv1-running.json index ee2e8c6083..6267acc38f 100644 --- a/src/webui/mock/mnist-tfv1-running.json +++ b/src/webui/mock/mnist-tfv1-running.json @@ -7,7 +7,7 @@ "id": "GD5MnU8G", "revision": 22, "execDuration": 172, - "logDir": "/home/user/nni/experiments/GD5MnU8G", + "logDir": "/home/user/nni-experiments/GD5MnU8G", "nextSequenceId": 3, "params": { "authorName": "default", diff --git a/src/webui/mock/mnist-tfv2-stage0.json b/src/webui/mock/mnist-tfv2-stage0.json index 833a8aff37..46d5e5439b 100644 --- a/src/webui/mock/mnist-tfv2-stage0.json +++ b/src/webui/mock/mnist-tfv2-stage0.json @@ -7,7 +7,7 @@ "id": "mw3lSbde", "revision": 3, "execDuration": 8, - "logDir": "/***/nni/experiments/mw3lSbde", + "logDir": "/***/nni-experiments/mw3lSbde", "nextSequenceId": 1, "params": { "authorName": "NNI Example", diff --git a/src/webui/mock/mnist-tfv2-stage1.json b/src/webui/mock/mnist-tfv2-stage1.json index 7e3f92d46c..36a03710f4 100644 --- a/src/webui/mock/mnist-tfv2-stage1.json +++ b/src/webui/mock/mnist-tfv2-stage1.json @@ -7,7 +7,7 @@ "id": "mw3lSbde", "revision": 34, "execDuration": 233, - "logDir": "/***/nni/experiments/mw3lSbde", + "logDir": "/***/nni-experiments/mw3lSbde", "nextSequenceId": 9, "params": { "authorName": "NNI Example", diff --git a/src/webui/mock/mnist-tfv2-stage2.json b/src/webui/mock/mnist-tfv2-stage2.json index 43a0f79035..d46a0f00e0 100644 --- a/src/webui/mock/mnist-tfv2-stage2.json +++ b/src/webui/mock/mnist-tfv2-stage2.json @@ -7,7 +7,7 @@ "id": "mw3lSbde", "revision": 39, "execDuration": 265, - "logDir": "/***/nni/experiments/mw3lSbde", + "logDir": "/***/nni-experiments/mw3lSbde", "nextSequenceId": 11, "params": { "authorName": "NNI Example", diff --git a/tools/README.md b/tools/README.md index 5fc3e7db7b..c1937f9736 100644 --- a/tools/README.md +++ b/tools/README.md @@ -4,7 +4,7 @@ The NNI CTL module is used to control Neural Network Intelligence, including sta ``` Ubuntu 16.04 or other Linux OS -python >= 3.5 +python >= 3.6 ``` ## Installation diff --git a/tools/bash-completion b/tools/bash-completion index 031f9b3476..918686b6b0 100644 --- a/tools/bash-completion +++ b/tools/bash-completion @@ -75,7 +75,7 @@ _nnictl() # add experiment IDs to candidates if desired if [[ $__nnictl_2nd_expid_cmds =~ " ${COMP_WORDS[1]} " ]]; then - local experiments=$(ls ~/nni/experiments 2>/dev/null) + local experiments=$(ls ~/nni-experiments 2>/dev/null) COMPREPLY+=($(compgen -W "$experiments" -- $cur)) fi diff --git a/tools/nni_annotation/__init__.py b/tools/nni_annotation/__init__.py index 84141ce8c7..a6e5ef66c1 100644 --- a/tools/nni_annotation/__init__.py +++ b/tools/nni_annotation/__init__.py @@ -122,7 +122,7 @@ def _expand_file_annotations(src_path, dst_path, nas_mode): def _generate_specific_file(src_path, dst_path, exp_id, trial_id, module): with open(src_path) as src, open(dst_path, 'w') as dst: try: - with open(os.path.expanduser('~/nni/experiments/%s/trials/%s/parameter.cfg'%(exp_id, trial_id))) as fd: + with open(os.path.expanduser('~/nni-experiments/%s/trials/%s/parameter.cfg'%(exp_id, trial_id))) as fd: para_cfg = json.load(fd) annotated_code = specific_code_generator.parse(src.read(), para_cfg["parameters"], module) if annotated_code is None: diff --git a/tools/nni_cmd/nnictl_utils.py b/tools/nni_cmd/nnictl_utils.py index 3fad64b1fc..3833c9317f 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/tools/nni_cmd/nnictl_utils.py @@ -11,7 +11,6 @@ import subprocess from functools import cmp_to_key from datetime import datetime, timezone -from pathlib import Path from subprocess import Popen from pyhdfs import HdfsClient from nni.package_utils import get_nni_installation_path @@ -445,9 +444,9 @@ def remote_clean(machine_list, experiment_id=None): sshKeyPath = machine.get('sshKeyPath') passphrase = machine.get('passphrase') if experiment_id: - remote_dir = '/' + '/'.join(['tmp', 'nni', 'experiments', experiment_id]) + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments', experiment_id]) else: - remote_dir = '/' + '/'.join(['tmp', 'nni', 'experiments']) + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments']) sftp = create_ssh_sftp_client(host, port, userName, passwd, sshKeyPath, passphrase) print_normal('removing folder {0}'.format(host + ':' + str(port) + remote_dir)) remove_remote_directory(sftp, remote_dir) @@ -517,11 +516,15 @@ def experiment_clean(args): print_warning('platform {0} clean up not supported yet.'.format(platform)) exit(0) #clean local data - home = str(Path.home()) - local_dir = nni_config.get_config('experimentConfig').get('logDir') - if not local_dir: - local_dir = os.path.join(home, 'nni', 'experiments', experiment_id) - local_clean(local_dir) + local_base_dir = nni_config.get_config('experimentConfig').get('logDir') + if not local_base_dir: + local_base_dir = NNI_HOME_DIR + local_experiment_dir = os.path.join(local_base_dir, experiment_id) + experiment_folder_name_list = ['checkpoint', 'db', 'log', 'trials'] + for folder_name in experiment_folder_name_list: + local_clean(os.path.join(local_experiment_dir, folder_name)) + if not os.listdir(local_experiment_dir): + local_clean(local_experiment_dir) experiment_config = Experiments() print_normal('removing metadata of experiment {0}'.format(experiment_id)) experiment_config.remove_experiment(experiment_id) diff --git a/tools/setup.py b/tools/setup.py index 48d6923dca..84f681ec2d 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -8,7 +8,7 @@ version = '999.0.0-developing', packages = setuptools.find_packages(exclude=['*test*']), - python_requires = '>=3.5', + python_requires = '>=3.6', install_requires = [ 'requests', 'ruamel.yaml',