From e0901b3e0913dbd284939666a763a0945dea941d Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Mon, 6 Jul 2020 12:17:05 +0800 Subject: [PATCH 1/4] using requirements.txt in nasbenchmark --- examples/nas/benchmarks/.gitignore | 3 +++ .../benchmarks/nasbench101.requirements.txt | 4 ++++ examples/nas/benchmarks/nasbench101.sh | 18 ++---------------- .../benchmarks/nasbench201.requirements.txt | 3 +++ examples/nas/benchmarks/nasbench201.sh | 15 ++------------- examples/nas/benchmarks/nds.requirements.txt | 2 ++ examples/nas/benchmarks/nds.sh | 14 ++------------ 7 files changed, 18 insertions(+), 41 deletions(-) create mode 100644 examples/nas/benchmarks/.gitignore create mode 100644 examples/nas/benchmarks/nasbench101.requirements.txt mode change 100644 => 100755 examples/nas/benchmarks/nasbench101.sh create mode 100644 examples/nas/benchmarks/nasbench201.requirements.txt mode change 100644 => 100755 examples/nas/benchmarks/nasbench201.sh create mode 100644 examples/nas/benchmarks/nds.requirements.txt mode change 100644 => 100755 examples/nas/benchmarks/nds.sh diff --git a/examples/nas/benchmarks/.gitignore b/examples/nas/benchmarks/.gitignore new file mode 100644 index 0000000000..daea239a5d --- /dev/null +++ b/examples/nas/benchmarks/.gitignore @@ -0,0 +1,3 @@ +nasbench_full.tfrecord +a.pth +data.zip diff --git a/examples/nas/benchmarks/nasbench101.requirements.txt b/examples/nas/benchmarks/nasbench101.requirements.txt new file mode 100644 index 0000000000..f84d2857fd --- /dev/null +++ b/examples/nas/benchmarks/nasbench101.requirements.txt @@ -0,0 +1,4 @@ +tensorflow==1.15.2 +tqdm +peewee +git+https://github.com/google-research/nasbench diff --git a/examples/nas/benchmarks/nasbench101.sh b/examples/nas/benchmarks/nasbench101.sh old mode 100644 new mode 100755 index 4b42628b81..1a6fe6a885 --- a/examples/nas/benchmarks/nasbench101.sh +++ b/examples/nas/benchmarks/nasbench101.sh @@ -1,22 +1,8 @@ set -e -mkdir -p /outputs /tmp - -echo "Installing dependencies..." -apt update && apt install -y wget git -pip install --no-cache-dir tqdm peewee - -echo "Installing NNI..." -cd /nni && echo "y" | source install.sh - -cd /tmp - -echo "Installing NASBench..." -git clone https://github.com/google-research/nasbench -cd nasbench && pip install -e . && cd .. echo "Downloading NAS-Bench-101..." wget https://storage.googleapis.com/nasbench/nasbench_full.tfrecord echo "Generating database..." -rm -f /outputs/nasbench101.db /outputs/nasbench101.db-journal -NASBENCHMARK_DIR=/outputs python -m nni.nas.benchmarks.nasbench101.db_gen nasbench_full.tfrecord +rm -f ${NASBENCHMARK_DIR}/nasbench101.db ${NASBENCHMARK_DIR}/nasbench101.db-journal +python -m nni.nas.benchmarks.nasbench101.db_gen nasbench_full.tfrecord diff --git a/examples/nas/benchmarks/nasbench201.requirements.txt b/examples/nas/benchmarks/nasbench201.requirements.txt new file mode 100644 index 0000000000..e36796143c --- /dev/null +++ b/examples/nas/benchmarks/nasbench201.requirements.txt @@ -0,0 +1,3 @@ +gdown +tqdm +peewee diff --git a/examples/nas/benchmarks/nasbench201.sh b/examples/nas/benchmarks/nasbench201.sh old mode 100644 new mode 100755 index 6121d95086..7531c8099e --- a/examples/nas/benchmarks/nasbench201.sh +++ b/examples/nas/benchmarks/nasbench201.sh @@ -1,19 +1,8 @@ set -e -mkdir -p /outputs /tmp - -echo "Installing dependencies..." -apt update && apt install -y wget -pip uninstall -y enum34 # https://github.com/iterative/dvc/issues/1995 -pip install --no-cache-dir gdown tqdm peewee - -echo "Installing NNI..." -cd /nni && echo "y" | source install.sh - -cd /tmp echo "Downloading NAS-Bench-201..." gdown https://drive.google.com/uc\?id\=1OOfVPpt-lA4u2HJrXbgrRd42IbfvJMyE -O a.pth echo "Generating database..." -rm -f /outputs/nasbench201.db /outputs/nasbench201.db-journal -NASBENCHMARK_DIR=/outputs python -m nni.nas.benchmarks.nasbench201.db_gen a.pth +rm -f ${NASBENCHMARK_DIR}/nasbench201.db ${NASBENCHMARK_DIR}/nasbench201.db-journal +python -m nni.nas.benchmarks.nasbench201.db_gen a.pth diff --git a/examples/nas/benchmarks/nds.requirements.txt b/examples/nas/benchmarks/nds.requirements.txt new file mode 100644 index 0000000000..5ca83aee88 --- /dev/null +++ b/examples/nas/benchmarks/nds.requirements.txt @@ -0,0 +1,2 @@ +tqdm +peewee diff --git a/examples/nas/benchmarks/nds.sh b/examples/nas/benchmarks/nds.sh old mode 100644 new mode 100755 index 3750a34514..069f9fa3a4 --- a/examples/nas/benchmarks/nds.sh +++ b/examples/nas/benchmarks/nds.sh @@ -1,19 +1,9 @@ set -e -mkdir -p /outputs /tmp - -echo "Installing dependencies..." -apt update && apt install -y wget zip -pip install --no-cache-dir tqdm peewee - -echo "Installing NNI..." -cd /nni && echo "y" | source install.sh - -cd /tmp echo "Downloading NDS..." wget https://dl.fbaipublicfiles.com/nds/data.zip -O data.zip unzip data.zip echo "Generating database..." -rm -f /outputs/nds.db /outputs/nds.db-journal -NASBENCHMARK_DIR=/outputs python -m nni.nas.benchmarks.nds.db_gen nds_data +rm -f ${NASBENCHMARK_DIR}/nds.db ${NASBENCHMARK_DIR}/nds.db-journal +python -m nni.nas.benchmarks.nds.db_gen nds_data From 842b931bb85aab948e8ac5431f28d14e71e78af4 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Mon, 6 Jul 2020 13:15:36 +0800 Subject: [PATCH 2/4] documentation and tests --- docs/en_US/NAS/Benchmarks.md | 30 ++++-------------------------- examples/nas/benchmarks/.gitignore | 1 + 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/docs/en_US/NAS/Benchmarks.md b/docs/en_US/NAS/Benchmarks.md index ffe9460edc..a2d80171b4 100644 --- a/docs/en_US/NAS/Benchmarks.md +++ b/docs/en_US/NAS/Benchmarks.md @@ -14,33 +14,11 @@ ## Data Preparation -To avoid storage and legal issues, we do not provide any prepared databases. We strongly recommend users to use docker to run the generation scripts, to ease the burden of installing multiple dependencies. Please follow the following steps. +To avoid storage and legal issues, we do not provide any prepared databases. Please follow the following steps. -**Step 1.** Clone NNI repo. Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.6`. - -```bash -git clone -b ${NNI_VERSION} https://github.com/microsoft/nni -``` - -**Step 2.** Run docker. - -For NAS-Bench-101, - -```bash -docker run -v ${HOME}/.nni/nasbenchmark:/outputs -v /path/to/your/nni:/nni tensorflow/tensorflow:1.15.2-py3 /bin/bash /nni/examples/nas/benchmarks/nasbench101.sh -``` - -For NAS-Bench-201, - -```bash -docker run -v ${HOME}/.nni/nasbenchmark:/outputs -v /path/to/your/nni:/nni ufoym/deepo:pytorch-cpu /bin/bash /nni/examples/nas/benchmarks/nasbench201.sh -``` - -For NDS, - -```bash -docker run -v ${HOME}/.nni/nasbenchmark:/outputs -v /path/to/your/nni:/nni python:3.7 /bin/bash /nni/examples/nas/benchmarks/nds.sh -``` +1. Enter `examples/nas/benchmarks`. +2. Install dependencies via `pip3 install -r xxx.requirements.txt`. `xxx` can be `nasbench101`, `nasbench201` or `nds`. +3. Generate the database via `./xxx.sh`. The directory that stores the benchmark file can be configured with `NASBENCHMARK_DIR` environment variable, which defaults to `~/.nni/nasbenchmark`. Please make sure there is at least 10GB free disk space and note that the conversion process can take up to hours to complete. diff --git a/examples/nas/benchmarks/.gitignore b/examples/nas/benchmarks/.gitignore index daea239a5d..be45d0a317 100644 --- a/examples/nas/benchmarks/.gitignore +++ b/examples/nas/benchmarks/.gitignore @@ -1,3 +1,4 @@ nasbench_full.tfrecord a.pth data.zip +nds_data From 818ab918beaf03030eb9a88bcb81766f0a3249fa Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 7 Jul 2020 11:11:55 +0800 Subject: [PATCH 3/4] Revert changes to docs --- docs/en_US/NAS/Benchmarks.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/en_US/NAS/Benchmarks.md b/docs/en_US/NAS/Benchmarks.md index a2d80171b4..75e3340b6c 100644 --- a/docs/en_US/NAS/Benchmarks.md +++ b/docs/en_US/NAS/Benchmarks.md @@ -14,11 +14,33 @@ ## Data Preparation -To avoid storage and legal issues, we do not provide any prepared databases. Please follow the following steps. +To avoid storage and legal issues, we do not provide any prepared databases. We strongly recommend users to use docker to run the generation scripts, to ease the burden of installing multiple dependencies. Please follow the following steps. -1. Enter `examples/nas/benchmarks`. -2. Install dependencies via `pip3 install -r xxx.requirements.txt`. `xxx` can be `nasbench101`, `nasbench201` or `nds`. -3. Generate the database via `./xxx.sh`. The directory that stores the benchmark file can be configured with `NASBENCHMARK_DIR` environment variable, which defaults to `~/.nni/nasbenchmark`. +**Step 1.** Clone NNI repo. Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.6`. + +```bash +git clone -b ${NNI_VERSION} https://github.com/microsoft/nni +``` + +**Step 2.** Run docker. + +For NAS-Bench-101, + +```bash +docker run -v ${HOME}/.nni/nasbenchmark:/outputs -v /path/to/your/nni:/nni tensorflow/tensorflow:1.15.2-py3 /bin/bash /nni/examples/nas/benchmarks/nasbench101.sh +``` + +For NAS-Bench-201, + +```bash +docker run -v ${HOME}/.nni/nasbenchmark:/outputs -v /path/to/your/nni:/nni ufoym/deepo:pytorch-cpu /bin/bash /nni/examples/nas/benchmarks/nasbench201.sh +``` + +For NDS, + +```bash +docker run -v ${HOME}/.nni/nasbenchmark:/outputs -v /path/to/your/nni:/nni python:3.7 /bin/bash /nni/examples/nas/benchmarks/nds.sh +``` Please make sure there is at least 10GB free disk space and note that the conversion process can take up to hours to complete. @@ -147,4 +169,4 @@ Here is a list of available operators used in NDS. .. autoclass:: nni.nas.benchmarks.nds.NdsTrialStats .. autoclass:: nni.nas.benchmarks.nds.NdsIntermediateStats -``` \ No newline at end of file +``` From ce513b3a408b1779230c21580e4d3c70a1df71c4 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 7 Jul 2020 20:41:51 +0800 Subject: [PATCH 4/4] Adding more checks to script --- examples/nas/benchmarks/nasbench101.sh | 13 ++++++++++++- examples/nas/benchmarks/nasbench201.sh | 13 ++++++++++++- examples/nas/benchmarks/nds.sh | 13 ++++++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/examples/nas/benchmarks/nasbench101.sh b/examples/nas/benchmarks/nasbench101.sh index 1a6fe6a885..b98ceb7772 100755 --- a/examples/nas/benchmarks/nasbench101.sh +++ b/examples/nas/benchmarks/nasbench101.sh @@ -1,8 +1,19 @@ +#!/bin/bash set -e +if [ -z "${NASBENCHMARK_DIR}" ]; then + NASBENCHMARK_DIR=~/.nni/nasbenchmark +fi + echo "Downloading NAS-Bench-101..." -wget https://storage.googleapis.com/nasbench/nasbench_full.tfrecord +if [ -f "nasbench_full.tfrecord" ]; then + echo "nasbench_full.tfrecord found. Skip download." +else + wget https://storage.googleapis.com/nasbench/nasbench_full.tfrecord +fi echo "Generating database..." rm -f ${NASBENCHMARK_DIR}/nasbench101.db ${NASBENCHMARK_DIR}/nasbench101.db-journal +mkdir -p ${NASBENCHMARK_DIR} python -m nni.nas.benchmarks.nasbench101.db_gen nasbench_full.tfrecord +rm -f nasbench_full.tfrecord diff --git a/examples/nas/benchmarks/nasbench201.sh b/examples/nas/benchmarks/nasbench201.sh index 7531c8099e..1a7feb8430 100755 --- a/examples/nas/benchmarks/nasbench201.sh +++ b/examples/nas/benchmarks/nasbench201.sh @@ -1,8 +1,19 @@ +#!/bin/bash set -e +if [ -z "${NASBENCHMARK_DIR}" ]; then + NASBENCHMARK_DIR=~/.nni/nasbenchmark +fi + echo "Downloading NAS-Bench-201..." -gdown https://drive.google.com/uc\?id\=1OOfVPpt-lA4u2HJrXbgrRd42IbfvJMyE -O a.pth +if [ -f "a.pth" ]; then + echo "a.pth found. Skip download." +else + gdown https://drive.google.com/uc\?id\=1OOfVPpt-lA4u2HJrXbgrRd42IbfvJMyE -O a.pth +fi echo "Generating database..." rm -f ${NASBENCHMARK_DIR}/nasbench201.db ${NASBENCHMARK_DIR}/nasbench201.db-journal +mkdir -p ${NASBENCHMARK_DIR} python -m nni.nas.benchmarks.nasbench201.db_gen a.pth +rm -f a.pth diff --git a/examples/nas/benchmarks/nds.sh b/examples/nas/benchmarks/nds.sh index 069f9fa3a4..9420287df8 100755 --- a/examples/nas/benchmarks/nds.sh +++ b/examples/nas/benchmarks/nds.sh @@ -1,9 +1,20 @@ +#!/bin/bash set -e +if [ -z "${NASBENCHMARK_DIR}" ]; then + NASBENCHMARK_DIR=~/.nni/nasbenchmark +fi + echo "Downloading NDS..." -wget https://dl.fbaipublicfiles.com/nds/data.zip -O data.zip +if [ -f "data.zip" ]; then + echo "data.zip found. Skip download." +else + wget https://dl.fbaipublicfiles.com/nds/data.zip -O data.zip +fi unzip data.zip echo "Generating database..." rm -f ${NASBENCHMARK_DIR}/nds.db ${NASBENCHMARK_DIR}/nds.db-journal +mkdir -p ${NASBENCHMARK_DIR} python -m nni.nas.benchmarks.nds.db_gen nds_data +rm -rf data.zip nds_data