Move functional tests to BrowserStack (DevExpress#1383)

* Move mobile tests to BrowserStack * Fix typo * leave only mobile functional tests * change ios to 9.2 * Try another ios device * Make linter happy * try another ios device * disable android device * increase some timeouts * change ios version * keep only problem tests * bring sl back * add desktop safari * Add android and desktop browsers * try to fix auth test * change android device * increase browser opening delay * try to use hub instead of delay * try to specify bs option * add "localIdentifier" * replace os.hostname with localhost * disable 'browserstack.local' * remove some tests * some changes in module * skip safari * add missed lines * return tests * only DevExpressgh-1311 * skip download test * localhost -> hostname * replace hub with delay * Try to clean connector up * compare android devices * Update browserstack package * some remarks * Revert test configuration files * Increase browser opening delay, add some info for browsers * use latest browsers * update "browserstack-connector" package * Change service port * fix config, update "browserstack-connector" package * Try to extend browser opening timeout
AndreyBelym · Apr 18, 2017 · 24da4de · 24da4de
1 parent 80ea625
commit 24da4de
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 83 deletions.
diff --git a/Gulpfile.js b/Gulpfile.js
@@ -546,8 +546,9 @@ gulp.task('publish-website', ['build-website-production'], function () {
 gulp.task('test-docs-travis', ['test-website-travis', 'lint']);
 
 
-function testFunctional (fixturesDir, testingEnvironmentName) {
+function testFunctional (fixturesDir, testingEnvironmentName, browserProviderName) {
     process.env.TESTING_ENVIRONMENT = testingEnvironmentName;
+    process.env.BROWSER_PROVIDER    = browserProviderName;
 
     return gulp
         .src(['test/functional/setup.js', fixturesDir + '/**/test.js'])
@@ -559,23 +560,23 @@ function testFunctional (fixturesDir, testingEnvironmentName) {
 }
 
 gulp.task('test-functional-travis-desktop-osx-and-ms-edge', ['build'], function () {
-    return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.saucelabsOSXDesktopAndMSEdgeBrowsers);
+    return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.osXDesktopAndMSEdgeBrowsers, functionalTestConfig.browserProviderNames.browserstack);
 });
 
 gulp.task('test-functional-travis-mobile', ['build'], function () {
-    return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.saucelabsMobileBrowsers);
+    return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.mobileBrowsers, functionalTestConfig.browserProviderNames.browserstack);
 });
 
 gulp.task('test-functional-local', ['build'], function () {
     return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.localBrowsers);
 });
 
 gulp.task('test-functional-travis-legacy', ['build'], function () {
-    return testFunctional('test/functional/legacy-fixtures', functionalTestConfig.testingEnvironmentNames.legacy);
+    return testFunctional('test/functional/legacy-fixtures', functionalTestConfig.testingEnvironmentNames.legacy, functionalTestConfig.browserProviderNames.sauceLabs);
 });
 
 gulp.task('test-functional-travis-old-browsers', ['build'], function () {
-    return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.oldBrowsers);
+    return testFunctional('test/functional/fixtures', functionalTestConfig.testingEnvironmentNames.oldBrowsers, functionalTestConfig.browserProviderNames.sauceLabs);
 });
 
 function getDockerEnv (machineName) {

diff --git a/package.json b/package.json
@@ -108,6 +108,7 @@
     "basic-auth": "^1.1.0",
     "body-parser": "^1.17.1",
     "broken-link-checker": "^0.7.0",
+    "browserstack-connector": "^0.1.4",
     "caller": "^1.0.1",
     "connect": "^3.4.0",
     "cross-spawn": "^4.0.0",

diff --git a/test/functional/config.js b/test/functional/config.js
@@ -4,80 +4,86 @@ var isTravisEnvironment = !!process.env.TRAVIS;
 var isCCNetEnvironment  = !!process.env.CCNET;
 var hostname            = isTravisEnvironment ? os.hostname() : '127.0.0.1';
 
+var browserProviderNames = {
+    sauceLabs:    'sauceLabs',
+    browserstack: 'browserstack'
+};
+
 var testingEnvironmentNames = {
-    saucelabsOSXDesktopAndMSEdgeBrowsers: 'saucelabs-osx-desktop-and-ms-edge-browsers',
-    saucelabsMobileBrowsers:              'saucelabs-mobile-browsers',
-    localBrowsers:                        'local-browsers',
-    oldBrowsers:                          'old-browsers',
-    legacy:                               'legacy'
+    osXDesktopAndMSEdgeBrowsers: 'osx-desktop-and-ms-edge-browsers',
+    mobileBrowsers:              'mobile-browsers',
+    localBrowsers:               'local-browsers',
+    oldBrowsers:                 'old-browsers',
+    legacy:                      'legacy'
 };
 
 var testingEnvironments = {};
 
-testingEnvironments[testingEnvironmentNames.saucelabsOSXDesktopAndMSEdgeBrowsers] = {
-    sauceLabs: {
-        username:  process.env.SAUCE_USERNAME_FUNCTIONAL_DESKTOP,
-        accessKey: process.env.SAUCE_ACCESS_KEY_FUNCTIONAL_DESKTOP,
-        jobName:   'functional tests - OS X desktop and MS edge browsers'
+testingEnvironments[testingEnvironmentNames.osXDesktopAndMSEdgeBrowsers] = {
+    jobName: 'functional tests - OS X desktop and MS edge browsers',
+
+    browserstack: {
+        username:  process.env.BROWSER_STACK_USERNAME,
+        accessKey: process.env.BROWSER_STACK_ACCESS_KEY
     },
 
     browsers: [
         {
-            platform:    'OS X 10.11',
-            browserName: 'safari',
-            version:     '9.0',
-            alias:       'safari'
+            os:        'OS X',
+            osVersion: 'Sierra',
+            name:      'safari',
+            alias:     'safari'
         },
         {
-            platform:    'OS X 10.11',
-            browserName: 'chrome',
-            alias:       'chrome-osx'
+            os:        'OS X',
+            osVersion: 'Sierra',
+            name:      'chrome',
+            alias:     'chrome-osx'
         },
         {
-            platform:    'OS X 10.11',
-            browserName: 'firefox',
-            alias:       'firefox-osx'
+            os:        'OS X',
+            osVersion: 'Sierra',
+            name:      'firefox',
+            alias:     'firefox-osx'
         },
         {
-            platform:    'Windows 10',
-            browserName: 'microsoftedge',
-            alias:       'edge',
-            version:     '13.10586'
+            os:        'Windows',
+            osVersion: '10',
+            name:      'edge',
+            alias:     'edge',
         }
     ]
 };
 
-testingEnvironments[testingEnvironmentNames.saucelabsMobileBrowsers] = {
-    sauceLabs: {
-        username:  process.env.SAUCE_USERNAME_FUNCTIONAL_MOBILE,
-        accessKey: process.env.SAUCE_ACCESS_KEY_FUNCTIONAL_MOBILE,
-        jobName:   'functional tests - mobile browsers'
+testingEnvironments[testingEnvironmentNames.mobileBrowsers] = {
+    jobName: 'functional tests - mobile browsers',
+
+    browserstack: {
+        username:  process.env.BROWSER_STACK_USERNAME,
+        accessKey: process.env.BROWSER_STACK_ACCESS_KEY
     },
 
     browsers: [
         {
-            platformName:    'Android',
-            deviceName:      'Android Emulator',
-            platformVersion: '5.1',
-            browserName:     'Browser',
-            alias:           'android'
+            os:        'android',
+            osVersion: '4.4',
+            device:    'Samsung Galaxy S5',
+            name:      'Android Browser',
+            alias:     'android'
         },
         {
-            // NOTE: we can't run tests on iOS 9.3 because of a bug in this version
-            // (see https://github.com/DevExpress/testcafe-hammerhead/issues/672#issuecomment-232043366).
-            // This bug is fixed in iOS 9.3.2 but it's not available on the farm.
-            platformName:    'iOS',
-            deviceName:      'iPad Retina',
-            platformVersion: '9.2',
-            browserName:     'Safari',
-            alias:           'ipad'
+            os:        'ios',
+            osVersion: '10.0',
+            device:    'iPad Pro (9.7 inch)',
+            name:      'Mobile Safari',
+            alias:     'ipad'
         },
         {
-            platformName:    'iOS',
-            deviceName:      'iPhone 6 Plus',
-            platformVersion: '9.2',
-            browserName:     'Safari',
-            alias:           'iphone'
+            os:        'ios',
+            osVersion: '10.0',
+            device:    'iPhone 7 Plus',
+            name:      'Mobile Safari',
+            alias:     'iphone'
         }
     ]
 };
@@ -104,10 +110,12 @@ testingEnvironments[testingEnvironmentNames.localBrowsers] = {
 };
 
 testingEnvironments[testingEnvironmentNames.oldBrowsers] = {
+    jobName: 'functional tests - ms desktop browsers',
+
     sauceLabs: {
         username:  process.env.SAUCE_USERNAME_FUNCTIONAL_DESKTOP,
         accessKey: process.env.SAUCE_ACCESS_KEY_FUNCTIONAL_DESKTOP,
-        jobName:   'functional tests - ms desktop browsers'
+
     },
 
     browsers: [
@@ -127,10 +135,11 @@ testingEnvironments[testingEnvironmentNames.oldBrowsers] = {
 };
 
 testingEnvironments[testingEnvironmentNames.legacy] = {
+    jobName: 'functional tests - legacy',
+
     sauceLabs: {
         username:  process.env.SAUCE_USERNAME_FUNCTIONAL_DESKTOP,
-        accessKey: process.env.SAUCE_ACCESS_KEY_FUNCTIONAL_DESKTOP,
-        jobName:   'functional tests - legacy'
+        accessKey: process.env.SAUCE_ACCESS_KEY_FUNCTIONAL_DESKTOP
     },
 
     browsers: [
@@ -159,6 +168,7 @@ module.exports = {
 
     testingEnvironmentNames: testingEnvironmentNames,
     testingEnvironments:     testingEnvironments,
+    browserProviderNames:    browserProviderNames,
 
     testCafe: {
         hostname: hostname,
@@ -174,5 +184,7 @@ module.exports = {
         port4:     3003
     },
 
+    browserstackConnectorServicePort: 4000,
+
     browsers: []
 };
diff --git a/test/functional/fixtures/regression/gh-845/test.js b/test/functional/fixtures/regression/gh-845/test.js
@@ -1,11 +1,11 @@
 // NOTE: we skip 'iphone,ipad' because no way to download file by link on these devices
 describe('[Regression](GH-845) Should execute click on a download link', function () {
     it('gh-845', function () {
-        return runTests('testcafe-fixtures/index-test.js', 'Click on a download link', { skip: 'iphone,ipad' });
+        return runTests('testcafe-fixtures/index-test.js', 'Click on a download link', { skip: ['iphone', 'ipad', 'android'] });
     });
 
     it('gh-845 in iframe', function () {
         return runTests('testcafe-fixtures/index-test.js', 'Click on a download link in iframe',
-            { selectorTimeout: 5000, skip: 'iphone,ipad' });
+            { selectorTimeout: 5000, skip: ['iphone', 'ipad', 'android'] });
     });
 });
diff --git a/test/functional/legacy-fixtures/regression/gh-845/test.js b/test/functional/legacy-fixtures/regression/gh-845/test.js
@@ -1,9 +1,12 @@
 describe('[Regression](GH-845) Should execute click on a download link', function () {
     it('gh-845', function () {
-        return runTests('testcafe-fixtures/index.test.js', 'Click on a download link');
+        return runTests('testcafe-fixtures/index.test.js', 'Click on a download link', { skip: ['android'] });
     });
 
     it('gh-845 in iframe', function () {
-        return runTests('testcafe-fixtures/index.test.js', 'Click on a download link in iframe', { selectorTimeout: 5000 });
+        return runTests('testcafe-fixtures/index.test.js', 'Click on a download link in iframe', {
+            selectorTimeout: 5000,
+            skip:            ['android']
+        });
     });
 });
diff --git a/test/functional/setup.js b/test/functional/setup.js
@@ -1,32 +1,36 @@
-var browserTools       = require('testcafe-browser-tools');
-var SauceLabsConnector = require('saucelabs-connector');
-var Promise            = require('pinkie');
-var caller             = require('caller');
-var path               = require('path');
-var createTestCafe     = require('../../lib');
-var config             = require('./config.js');
-var site               = require('./site');
-var getTestError       = require('./get-test-error.js');
+var browserTools   = require('testcafe-browser-tools');
+var SlConnector    = require('saucelabs-connector');
+var BsConnector    = require('browserstack-connector');
+var Promise        = require('pinkie');
+var caller         = require('caller');
+var path           = require('path');
+var createTestCafe = require('../../lib');
+var config         = require('./config.js');
+var site           = require('./site');
+var getTestError   = require('./get-test-error.js');
 
 var testCafe     = null;
 var browsersInfo = null;
 
-var slConnector = null;
-var slBrowsers  = null;
+var connector        = null;
+var browserInstances = null;
 
 const WAIT_FOR_FREE_MACHINES_REQUEST_INTERVAL  = 60000;
 const WAIT_FOR_FREE_MACHINES_MAX_ATTEMPT_COUNT = 60;
 
+const BROWSER_OPENING_TIMEOUT = 90000;
+
 const FUNCTIONAL_TESTS_SELECTOR_TIMEOUT  = 200;
 const FUNCTIONAL_TESTS_ASSERTION_TIMEOUT = 1000;
 
-var envName     = process.env.TESTING_ENVIRONMENT || config.testingEnvironmentNames.localBrowsers;
-var environment = config.testingEnvironments[envName];
+var envName         = process.env.TESTING_ENVIRONMENT || config.testingEnvironmentNames.localBrowsers;
+var environment     = config.testingEnvironments[envName];
+var browserProvider = process.env.BROWSER_PROVIDER;
+var isBrowserStack  = browserProvider === config.browserProviderNames.browserstack;
 
 config.browsers = environment.browsers;
 
-const SAUCE_LABS_REQUESTED_MACHINES_COUNT = environment.browsers.length;
-
+const REQUESTED_MACHINES_COUNT = environment.browsers.length;
 
 function getBrowserInfo (settings) {
     return testCafe
@@ -48,24 +52,33 @@ function initBrowsersInfo () {
 }
 
 function openRemoteBrowsers () {
-    slConnector = new SauceLabsConnector(environment.sauceLabs.username, environment.sauceLabs.accessKey);
+    var Connector = isBrowserStack ? BsConnector : SlConnector;
+
+    connector = new Connector(environment[browserProvider].username, environment[browserProvider].accessKey,
+        { servicePort: config.browserstackConnectorServicePort });
 
-    return slConnector
+    return connector
         .connect()
         .then(function () {
-            return slConnector.waitForFreeMachines(SAUCE_LABS_REQUESTED_MACHINES_COUNT,
+            return connector.waitForFreeMachines(REQUESTED_MACHINES_COUNT,
                 WAIT_FOR_FREE_MACHINES_REQUEST_INTERVAL, WAIT_FOR_FREE_MACHINES_MAX_ATTEMPT_COUNT);
         })
         .then(function () {
+            var buildInfo = {
+                jobName: environment.jobName,
+                build:   process.env.TRAVIS_BUILD_ID || '',
+                tags:    [process.env.TRAVIS_BRANCH || 'master']
+            };
+
             var openBrowserPromises = browsersInfo.map(function (browserInfo) {
-                return slConnector.startBrowser(browserInfo.settings, browserInfo.connection.url,
-                    environment.sauceLabs.jobName);
+                return connector.startBrowser(browserInfo.settings, browserInfo.connection.url, buildInfo,
+                    { openingTimeout: BROWSER_OPENING_TIMEOUT });
             });
 
             return Promise.all(openBrowserPromises);
         })
         .then(function (browsers) {
-            slBrowsers = browsers;
+            browserInstances = browsers;
         });
 }
 
@@ -81,13 +94,13 @@ function openLocalBrowsers () {
 }
 
 function closeRemoteBrowsers () {
-    var closeBrowserPromises = slBrowsers.map(function (browser) {
-        return slConnector.stopBrowser(browser);
+    var closeBrowserPromises = browserInstances.map(function (browser) {
+        return connector.stopBrowser(isBrowserStack ? browser.id : browser);
     });
 
     return Promise.all(closeBrowserPromises)
         .then(function () {
-            return slConnector.disconnect();
+            return connector.disconnect();
         });
 }