Tweak unit/e2e tests and coverage

- Combine code coverage from unit and integration tests. Now we know that
  the full code coverage for iris is really 48% instead of like 25%. This
  is done through `make combined-cov` which travis now runs.

- Bail from the e2e coverage tests if we can't start the coverage server,
  which will be the case if iris-api is already running outside of the e2e
  tests. (Common on local development)

- Akin to that, kill the gunicorn process so the coverage tests are able to
  spawn another instance of iris-api which will keep track of the code paths
  which are run.

- Fix a gevent issue (we weren't monkey patching) in the test server that
  is ran during the e2e code coverage. This was causing some of the tests
  to hang.

- Skip the notification RPC test if sender is not running.

- Show the e2etests as they run so we can know which tests were skipped by
  looking at the jenkins logs.

.ci/after_success.sh

@@ -3,5 +3,9 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common.sh"
 
 pushd ${TRAVIS_BUILD_DIR}
-make unit-cov
-make e2e-cov
+
+# Kill the `make serve` executed prior to this, so the e2e coverage tests
+# will properly run. The sender will still be alive.
+killall -9 gunicorn
+
+make combined-cov

Makefile

@@ -23,9 +23,16 @@ check:
 	make test
 
 unit-cov:
-	py.test --cov-report term-missing --cov=iris test
+	COVERAGE_FILE=.coverage.unit py.test --cov-report term-missing --cov=iris test
 
 e2e-cov:
 	./test/e2etest_coverage.sh
 
+combined-cov:
+	rm -f .coverage*
+	make unit-cov
+	SUPPORT_COMBINED_COVERAGE=1 make e2e-cov
+	coverage combine
+	coverage report -m
+
 .PHONY: test

src/iris/bin/run_server.py

@@ -6,6 +6,8 @@
 # -*- coding:utf-8 -*-
 
 import gevent
+from gevent import monkey
+monkey.patch_all()  # noqa
 from gevent.pywsgi import WSGIServer
 from gevent.subprocess import call
 

test/e2etest.py

@@ -12,9 +12,11 @@
 import iris.bin.iris_ctl as iris_ctl
 from click.testing import CliRunner
 import uuid
+import socket
 
 
 server = 'http://localhost:16649/'
+sender_address = ('localhost', 2321)
 base_url = server + 'v0/'
 ui_url = server
 
@@ -1700,6 +1702,16 @@ def test_app_stats(sample_application_name):
 
 
 def test_post_notification(sample_user, sample_application_name):
+
+    # The iris-api in this case will send a request to iris-sender's
+    # rpc endpoint. Don't bother if sender isn't working.
+    try:
+        sock = socket.socket()
+        sock.connect(sender_address)
+        sock.close()
+    except socket.error:
+        pytest.skip('Skipping this test as sender is not running/reachable.')
+
     re = requests.post(base_url + 'notifications', json={})
     assert re.status_code == 400
     assert 'Missing required atrributes' in re.text

test/e2etest_coverage.sh

@@ -1,10 +1,19 @@
 #!/bin/bash
 CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd $(dirname ${CURDIR})
+if [ ! -z "$SUPPORT_COMBINED_COVERAGE" ]; then
+  export COVERAGE_FILE=.coverage.e2e
+fi
 coverage run --source=iris src/iris/bin/run_server.py configs/config.dev.yaml &>/dev/null &
 pid=$!
 sleep 2
-py.test test/e2etest.py
+if ! kill -0 "$pid" ; then
+  echo Server failed to start. Bailing.
+  exit 1
+fi
+py.test -v test/e2etest.py
 sleep 1
 /bin/kill -INT $pid
-coverage report -m
+if [ -z "$SUPPORT_COMBINED_COVERAGE" ]; then
+  coverage report -m
+fi