adding app tests

fixes #12

making it easy to do full app testing which includes being able to spin
up a browser and verify you can run various juttle programs through
outrigger

gulpfile.js

@@ -64,16 +64,44 @@ gulp.task('instrument', function () {
     .pipe(istanbul.hookRequire());
 });
 
+function gulp_test_app() {
+    return gulp.src([
+    ])
+    .pipe(mocha({
+        log: true,
+        timeout: 5000,
+        reporter: 'spec',
+        ui: 'bdd',
+        ignoreLeaks: true,
+        globals: ['should']
+    }));
+}
+
 function gulp_test() {
-    return gulp.src('test/**/*.spec.js')
-        .pipe(mocha({
-            log: true,
-            timeout: 5000,
-            reporter: 'spec',
-            ui: 'bdd',
-            ignoreLeaks: true,
-            globals: ['should']
-        }));
+    var argv = require('yargs').argv;
+    var tests = [
+        'test/**/*.spec.js'
+    ];
+
+    // by passing the argument `--app` you can also run the app tests
+    // which require spinning up a browser, by default we do not run
+    // the app tests
+    if (!argv.app) {
+        tests.push(
+            // exclude app tests
+            '!test/app/**/*.spec.js'
+        );
+    }
+
+    return gulp.src(tests)
+    .pipe(mocha({
+        log: true,
+        timeout: 5000,
+        reporter: 'spec',
+        ui: 'bdd',
+        ignoreLeaks: true,
+        globals: ['should']
+    }));
 }
 
 gulp.task('test', function() {
@@ -82,17 +110,17 @@ gulp.task('test', function() {
 
 gulp.task('test-coverage', ['instrument'], function() {
     return gulp_test()
-        .pipe(istanbul.writeReports())
-        .pipe(istanbul.enforceThresholds({
-            thresholds: {
-                global: {
-                    statements: 71,
-                    branches: 63,
-                    functions: 61,
-                    lines: 55
-                }
+    .pipe(istanbul.writeReports())
+    .pipe(istanbul.enforceThresholds({
+        thresholds: {
+            global: {
+                statements: 76,
+                branches: 71,
+                functions: 69,
+                lines: 73 
             }
-        }));
+        }
+    }));
 });
 
 gulp.task('default', ['test', 'lint']);

lib/service-juttled.js

@@ -17,7 +17,7 @@ var API_PREFIX = '/api/v0';
 
 var JuttledService = Base.extend({
 
-    initialize: function(options) {
+    initialize: function(options, cb) {
         var config = read_config(options);
         Juttle.adapters.load(config.adapters);
         var self = this;
@@ -52,6 +52,7 @@ var JuttledService = Base.extend({
 
         this._server = this._app.listen(options.port, function() {
             logger.info('Juttled service listening at http://localhost:' + options.port + " with root directory:" + self._root_directory);
+            cb && cb();
         });
     },
 

package.json

@@ -82,15 +82,19 @@
     "gulp-mocha": "^2.1.3",
     "isparta": "^4.0.0",
     "json-loader": "^0.5.4",
+    "nconf": "^0.8.2",
     "node-libs-browser": "^0.5.3",
     "node-sass": "^3.4.2",
+    "phantomjs": "^1.9.19",
     "resolve-url-loader": "^1.4.2",
     "rimraf": "^2.4.4",
     "sass-loader": "^3.1.1",
+    "sauce-connect-launcher": "^0.14.0",
     "selenium-webdriver": "^2.48.2",
     "style-loader": "^0.13.0",
     "webpack": "^1.12.6",
-    "webpack-dev-middleware": "^1.2.0"
+    "webpack-dev-middleware": "^1.2.0",
+    "yargs": "^3.31.0"
   },
   "engines": {
     "node": ">=4.2.0",

test/README.md

@@ -0,0 +1,53 @@
+# Test
+
+## Running Unit Tests
+
+To run the built in unit tests simply run:
+
+```
+gulp test
+```
+
+## Running App Tests
+
+To run the unit tests and app tests that uses your local `google-chrome` browser
+to run the tests:
+
+```
+gulp test --app
+```
+
+The above relies on you having the right version of chrome. To avoid version
+problems with chrome you can use docker for testing like so:
+
+```
+docker run -d -p 4444:4444 --name selenium-hub selenium/hub
+docker run -d --link selenium-hub:hub selenium/node-chrome
+```
+
+Then when running the tests you have to tell selenium to hit the selenium
+hub running on the `selenium-hub` container and the tests to hit the 
+host ip address which is usually `172.17.42.1` but you can check what your
+host ip address is with:
+
+```
+docker network inspect bridge | grep Gateway
+```
+
+That `Gateway` is your host ip that should be used with the following command:
+
+```
+OUTRIGGER_HOST=172.17.42.1 SELENIUM_REMOTE_URL='http://localhost:4444/wd/hub' gulp test --app
+```
+
+That will run the same app tests through the docker selenium setup and verify
+everything is working as expected.
+
+## Helper Scripts
+
+To simplify bringing up the selenium setup and tearing it down you can use
+the helper scripts under `test/scripts`:
+
+ * `test/scripts/start_selenium_setup.sh`
+ * `test/scripts/stop_selenium_setup.sh`
+

test/app/README.md

@@ -1,11 +1,6 @@
-These tests test the full flow of loading a juttle file into outriggerd, filling out inputs if there are any, and verifying that the program output is correct.
+# Outrigger App Tests
 
-They are not fully automated yet and require some manual startup of services.
-
-* Install [ChromeDriver](https://sites.google.com/a/chromium.org/chromedriver/downloads) and start it (starts on default port 9515)
-* Make sure outriggerd is running
-
-To run the tests:
-```
-mocha app.spec.js
-```
+These tests verify the integration of all juttle components within the outrigger
+application and are not executed when you do the regular `gulp test` but instead
+you have to run `gulp test:app` to run these since they will take over your
+desktop with a chrome browser and run the end to end tests.

test/app/app.spec.js

@@ -1,56 +1,59 @@
-"use strict";
+'use strict';
 
 let DemoAppTester = require('./lib/demo-app-tester');
-let webdriver = require('selenium-webdriver');
 let expect = require('chai').expect;
 let path = require('path');
 
-const TEST_TIMEOUT = 10000;
-const BROWSER = "from";
-const WEBDRIVER_URL = "http://localhost:9515/";
+const TEST_TIMEOUT = 30000;
 
-
-// skipped while webdriver is not properly configured in travis
-describe.skip("demo-app", function() {
+describe('demo-app', function() {
     this.timeout(TEST_TIMEOUT);
-    let driver;
     let demoAppTester;
-    beforeEach(() => {
-        driver = new webdriver.Builder()
-            .forBrowser(BROWSER)
-            .usingServer(WEBDRIVER_URL)
-            .build();
-        demoAppTester = new DemoAppTester(driver);
+
+    before(function(done) {
+        demoAppTester = new DemoAppTester();
+        demoAppTester.start(done);
     });
 
-    it("open juttle program with no inputs", () => {
-        return demoAppTester.loadFile(path.join(__dirname, "juttle", "no-inputs.juttle"))
-           .then(() => {
-               return demoAppTester.getLoggerOutput('myLogger');
-           })
-           .then((value) => {
-               expect(JSON.parse(value)).to.deep.equal([{time: new Date(0).toISOString(), v: 10}]);
-           });
+    after(() => {
+        demoAppTester.stop();
     });
 
-    it("open juttle program with an input, fill it out, and run", () => {
-        return demoAppTester.loadFile(path.join(__dirname, "juttle", "one-input.juttle"))
-            .then(() => {
-                return demoAppTester.findInputControl("a");
-            })
-            .then((inputElem) => {
-                inputElem.sendKeys("AAA");
-            })
-            .then(demoAppTester.clickPlay)
-            .then(() => {
-                return demoAppTester.getLoggerOutput('myLogger');
-            })
-            .then((value) => {
-                expect(JSON.parse(value)).to.deep.equal([{time: new Date(0).toISOString(), v: "AAA"}]);
-            });
+    it('open juttle program with no inputs', () => {
+        return demoAppTester.run({
+            path: path.join(__dirname, 'juttle', 'no-inputs.juttle')
+        })
+        .then(() => {
+            return demoAppTester.getTextOutput('output');
+        })
+        .then((value) => {
+            expect(JSON.parse(value)).to.deep.equal([
+                { time: new Date(0).toISOString(), value: 10 }
+            ]);
+        });
     });
 
-    afterEach(() => {
-        driver.quit();
+    it('open juttle program with an input, fill it out, and run', () => {
+        return demoAppTester.run({
+            path: path.join(__dirname, 'juttle', 'one-input.juttle')
+        })
+        .then(() => {
+            return demoAppTester.findInputControl('a');
+        })
+        .then((inputElem) => {
+            inputElem.sendKeys('AAA');
+        })
+        .then(() => {
+            demoAppTester.clickPlay();
+        })
+        .then(() => {
+            return demoAppTester.getTextOutput('myLogger');
+        })
+        .then((value) => {
+            expect(JSON.parse(value)).to.deep.equal([
+                { time: new Date(0).toISOString(), value: 'AAA' }
+            ]);
+        });
     });
+
 });

test/app/juttle/no-inputs.juttle

@@ -1,3 +1,3 @@
 emit -from :0: -limit 1
-| put v = 10
-| @logger -display.style 'json' -title 'myLogger'
+| put value = 10
+| view text -format 'json' -title 'output'

test/app/juttle/one-input.juttle

@@ -1,4 +1,5 @@
 input a: text -label 'a';
+
 emit -from :0: -limit 1
-| put v = a
-| @logger -display.style 'json' -title 'myLogger'
+| put value = a
+| view text -format 'json' -title 'myLogger'