Refactor integration tests (#2190)

azure-pipelines.yml

@@ -52,24 +52,12 @@ jobs:
     displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      python3 naive_test.py
-    displayName: 'Naive test'
-  - script: |
-      cd test
-      python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      python3 metrics_test.py
-    displayName: 'Trial job metrics test'
-  - script: |
-      cd test
-      python3 cli_test.py
-    displayName: 'nnicli test'
+      python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+    displayName: 'Simple test'
   - script: |
       cd docs/en_US/
       sphinx-build -M html . _build -W
@@ -101,20 +89,12 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      python3 naive_test.py
-    displayName: 'Naive test'
-  - script: |
-      cd test
-      python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      python3 cli_test.py
-    displayName: 'nnicli test'
+      python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+    displayName: 'Simple test'
 
 - job: 'basic_test_pr_Windows'
   pool:
@@ -137,13 +117,9 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       cd test
-      powershell.exe -file unittest.ps1
+      powershell.exe -file scripts/unittest.ps1
     displayName: 'unit test'
   - script: |
       cd test
-      python tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
-    displayName: 'nnicli test'
+      python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+    displayName: 'Simple test'

examples/model_compress/model_prune_torch.py

@@ -67,7 +67,7 @@
         'dataset_name': 'cifar10',
         'model_name': 'vgg16',
         'pruner_class': ActivationMeanRankFilterPruner,
-        'configure_list': [{
+        'config_list': [{
             'sparsity': 0.5,
             'op_types': ['default'],
             'op_names': ['feature.0', 'feature.24', 'feature.27', 'feature.30', 'feature.34', 'feature.37']

test/config_test/tuners/mnist-curvefitting.test.yml → test/config/assessors/curvefitting.yml

@@ -1,9 +1,9 @@
 authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
-maxTrialNum: 2
-trialConcurrency: 1
-searchSpacePath: search_space.json
+maxTrialNum: 8
+trialConcurrency: 8
+searchSpacePath: ../naive_trial/search_space.json
 
 tuner:
   builtinTunerName: TPE
@@ -17,12 +17,12 @@ assessor:
       start_step: 6
       threshold: 0.95
 trial:
-  codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  codeDir: ../naive_trial
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
 multiPhase: false
 multiThread: false
 
-trainingServicePlatform: local
+trainingServicePlatform: local

test/config_test/multi_phase/multi_phase_tpe.test.yml → test/config/assessors/medianstop.yml

@@ -2,21 +2,24 @@ authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
 maxTrialNum: 8
-trialConcurrency: 4
-searchSpacePath: ./search_space.json
+trialConcurrency: 8
+searchSpacePath: ../naive_trial/search_space.json
 
 tuner:
   builtinTunerName: TPE
   classArgs:
     optimize_mode: maximize
 
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+
 trial:
-  codeDir: .
-  command: python3 multi_phase.py
+  codeDir: ../naive_trial
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
-multiPhase: true
-multiThread: false
 
 trainingServicePlatform: local

test/config_test/examples/mnist-annotation.test.yml → test/config/examples/mnist-annotation.yml

@@ -12,7 +12,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-annotation
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: true

test/config_test/examples/mnist-nested-search-space.test.yml → test/config/examples/mnist-nested-search-space.yml

@@ -14,7 +14,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-nested-search-space
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-pytorch.test.yml → test/config/examples/mnist-pytorch.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-pytorch
-  command: python3 mnist.py  --epochs 2
+  command: python3 mnist.py  --epochs 1
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-tfv1.test.yml → test/config/examples/mnist-tfv1.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false