diff --git a/azure-pipelines.test.yml b/azure-pipelines.test.yml
new file mode 100644
index 0000000000..0879eb1a10
--- /dev/null
+++ b/azure-pipelines.test.yml
@@ -0,0 +1,72 @@
+trigger:
+- dev-it
+
+jobs:
+
+- job: 'Ubuntu_16_04'
+  pool: 'NNI CI GPU'
+
+  steps:
+  - script: python3 -m pip install --upgrade pip setuptools
+    displayName: 'Install python tools'
+  - script: |
+      source install.sh
+    displayName: 'Install nni toolkit via source code'
+  - script: |
+      python3 -m pip install sklearn --user
+      python3 -m pip install torchvision --user
+      python3 -m pip install keras --user
+      python3 -m pip install tensorflow==1.9.0 --user
+    displayName: 'Install dependencies for integration tests'
+  - script: |
+      cd test
+      source unittest.sh
+    displayName: 'Unit test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 naive_test.py
+    displayName: 'Naive test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
+    displayName: 'Built-in tuners / assessors tests'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --config config_test/examples/mnist.test.yml
+    displayName: 'Examples on local machine tests'
+  
+- job: 'macOS_10_13'
+  pool:
+    vmImage: 'macOS 10.13'
+  strategy:
+    matrix:
+      Python36:
+        PYTHON_VERSION: '3.6'
+
+  steps:
+  - script: python3 -m pip install --upgrade pip setuptools
+    displayName: 'Install python tools'
+  - script: |
+      source install.sh
+    displayName: 'Install nni toolkit via source code'
+  - script: |
+      python3 -m pip install sklearn --user
+      python3 -m pip install torchvision --user
+      python3 -m pip install keras --user
+    displayName: 'Install dependencies for integration tests'
+  - script: |
+      cd test
+      PATH=$HOME/Library/Python/3.7/bin:$PATH && source unittest.sh
+    displayName: 'Unit test'
+  - script: |
+      cd test
+      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 naive_test.py
+    displayName: 'Naive test'
+  - script: |
+      cd test
+      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 tuner_test.py
+    displayName: 'Built-in tuners / assessors tests'
+  - script: |
+      cd test
+      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 config_test.py --ts local
+    displayName: 'Examples on local machine tests'
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 90df728c8a..da1b3e80b2 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -20,10 +20,6 @@ jobs:
       python3 -m pip install keras --user
       python3 -m pip install tensorflow==1.9.0 --user
     displayName: 'Install dependencies for integration tests'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --config config_test/examples/mnist.test.yml
-    displayName: 'Examples on local machine tests'
   - script: |
       cd test
       source unittest.sh
@@ -36,7 +32,11 @@ jobs:
       cd test
       PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
-
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --config config_test/examples/mnist.test.yml
+    displayName: 'Examples on local machine tests'
+  
 - job: 'macOS_10_13'
   pool:
     vmImage: 'macOS 10.13'