From 5b638cb3516793c17afcb7e3f4ed95fe3c17ce7b Mon Sep 17 00:00:00 2001
From: Bill Robinson <dseevr@users.noreply.github.com>
Date: Tue, 3 Jan 2017 13:39:59 -0800
Subject: [PATCH 1/2] systemtests now run completely in containers

Signed-off-by: Bill Robinson <dseevr@users.noreply.github.com>
---
 Makefile                            |   9 +-
 build/Dockerfile.base               |  18 ++++
 build/Dockerfile.build              |  15 +--
 build/Dockerfile.systemtests        |   9 ++
 scripts/build.sh                    |   3 +
 scripts/generate-certificate.sh     |   0
 scripts/run.sh                      |  30 ------
 scripts/systemtests.sh              | 161 ++++++++++++++++++++++++++++
 scripts/systemtests_in_container.sh |  19 ++++
 systemtests/auth_test.go            |   4 +-
 systemtests/basic_test.go           |  11 +-
 systemtests/init_test.go            |  91 ++++++----------
 systemtests/local_user_test.go      |  11 +-
 systemtests/mock_server.go          |   2 +-
 systemtests/rbac_test.go            |  27 +++--
 15 files changed, 273 insertions(+), 137 deletions(-)
 create mode 100644 build/Dockerfile.base
 create mode 100644 build/Dockerfile.systemtests
 mode change 100644 => 100755 scripts/generate-certificate.sh
 delete mode 100644 scripts/run.sh
 create mode 100755 scripts/systemtests.sh
 create mode 100755 scripts/systemtests_in_container.sh

diff --git a/Makefile b/Makefile
index 54da7a9e..33a3a01f 100644
--- a/Makefile
+++ b/Makefile
@@ -13,8 +13,10 @@ build: checks
 checks:
 	@bash ./scripts/checks.sh
 
-# ci does everything necessary for a Github PR-triggered CI run
-ci: checks test
+# ci does everything necessary for a Github PR-triggered CI run.
+# currently, this means building a container image and running
+# all of the available tests. (systemtests require a container)
+ci: build test
 
 # generate-certificate generates a local key and cert for running the proxy.
 # if an existing certificate and key exist, it will do nothing.
@@ -35,8 +37,7 @@ run: build generate-certificate
 
 # systemtests runs the system tests suite.
 systemtests:
-	go get gopkg.in/check.v1
-	go test -v -timeout 5m ./systemtests -check.v
+	@bash ./scripts/systemtests.sh
 
 # unittests runs all the unit tests
 unit-tests:
diff --git a/build/Dockerfile.base b/build/Dockerfile.base
new file mode 100644
index 00000000..371c4834
--- /dev/null
+++ b/build/Dockerfile.base
@@ -0,0 +1,18 @@
+FROM ubuntu:16.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get -y upgrade && \
+    	    apt-get -y install curl build-essential docker.io
+
+RUN curl -O https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz
+RUN tar -C /usr/local -xzf go1.7.4.linux-amd64.tar.gz && rm go1.7.4.linux-amd64.tar.gz
+
+ENV PATH="/usr/local/go/bin:$PATH"
+ENV GOPATH="/go"
+
+RUN mkdir -p /go/src/github.com/contiv/ccn_proxy
+
+WORKDIR /go/src/github.com/contiv/ccn_proxy
+
+ENTRYPOINT ["/bin/bash"]
diff --git a/build/Dockerfile.build b/build/Dockerfile.build
index d71b9c2d..f3cedf64 100644
--- a/build/Dockerfile.build
+++ b/build/Dockerfile.build
@@ -1,20 +1,7 @@
-FROM ubuntu:16.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get -y upgrade && \
-    	    apt-get -y install curl build-essential docker.io
-
-RUN curl -O https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz
-RUN tar -C /usr/local -xzf go1.7.4.linux-amd64.tar.gz && rm go1.7.4.linux-amd64.tar.gz
-
-ENV PATH="/usr/local/go/bin:$PATH"
-ENV GOPATH="/go"
+FROM ccn_proxy_build_base
 
 COPY ./ /go/src/github.com/contiv/ccn_proxy
 
-WORKDIR /go/src/github.com/contiv/ccn_proxy
-
 RUN scripts/build-in-container.sh
 
 ENTRYPOINT ["docker", "build"]
diff --git a/build/Dockerfile.systemtests b/build/Dockerfile.systemtests
new file mode 100644
index 00000000..8c81ca7f
--- /dev/null
+++ b/build/Dockerfile.systemtests
@@ -0,0 +1,9 @@
+FROM ccn_proxy_build_base
+
+# go-check must be installed to run systemtests
+RUN go get gopkg.in/check.v1
+
+COPY ./ /go/src/github.com/contiv/ccn_proxy
+
+ENTRYPOINT ["/bin/bash"]
+#, "./scripts/systemtests_in_container.sh"]
diff --git a/scripts/build.sh b/scripts/build.sh
index 09e2788b..e1b1296f 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -41,6 +41,9 @@ fi
 
 cd $START_DIR
 
+# ensure base has been built
+docker build -t "ccn_proxy_build_base" -f ./build/Dockerfile.base .
+
 # create a local build image
 docker build -t $BUILD_IMAGE_NAME -f ./build/Dockerfile.build .
 
diff --git a/scripts/generate-certificate.sh b/scripts/generate-certificate.sh
old mode 100644
new mode 100755
diff --git a/scripts/run.sh b/scripts/run.sh
deleted file mode 100644
index ef3bfa72..00000000
--- a/scripts/run.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -euxo pipefail
-
-# if DOCKER_HOST is not set, just run the image locally.
-# otherwise, scp the key and cert over to the docker machine before starting it.
-if [ -z "${DOCKER_HOST-}" ]; then
-    echo "Running on local machine:"
-    docker run --rm \
-	   -p 9999:9999 \
-	   -v $PWD/local_certs:/certs ccn_proxy:devbuild \
-	   --tls-key-file=/certs/local.key \
-	   --tls-certificate=/certs/cert.pem \
-	   --data-store-address=etcd://localhost:2379
-else
-    echo "Copying certificates to docker-machine:"
-    cert_path="/tmp/ccn_proxy/"
-
-    docker-machine ssh $DOCKER_MACHINE_NAME mkdir -p $cert_path
-    docker-machine scp './local_certs/local.key' $DOCKER_MACHINE_NAME:$cert_path
-    docker-machine scp './local_certs/cert.pem'  $DOCKER_MACHINE_NAME:$cert_path
-
-    echo "Running on Docker Machine:"
-    docker run --rm \
-	   -p 9999:9999 \
-	   -v $cert_path:/certs ccn_proxy:devbuild \
-	   --tls-key-file=/certs/local.key \
-	   --tls-certificate=/certs/cert.pem \
-	   --data-store-address=etcd://localhost:2379
-fi
diff --git a/scripts/systemtests.sh b/scripts/systemtests.sh
new file mode 100755
index 00000000..c79465c2
--- /dev/null
+++ b/scripts/systemtests.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+
+#
+# systemtests.sh is invoked by the `make systemtests` target.
+# It is responsible for setting up all the containers necessary for running our
+# systemtest suite and running the tests themselves in a container against them.
+#
+# It does the following:
+#
+#  1. builds a systemtests container
+#  2. creates a docker network (all containers are attached to it)
+#  3. starts an etcd container
+#  4. starts a consul container
+#  5. starts a systemtests container (does nothing by default)
+#  6. starts a ccn_proxy container on port 10000 linked to etcd
+#  7. starts a ccn_proxy container on port 10001 linked to consul
+#  8. executes ./scripts/systemtests_in_container.sh which runs all the systemtests
+#     against the etcd proxy and consul proxy
+#  9. stops etcd proxy container
+# 10. stops consul proxy container
+# 11. stops systemtests container
+# 12. stops etcd container
+# 13. stops consul container
+# 14. destroys the docker network
+#
+
+set -euo pipefail
+
+IMAGE_NAME="ccn_proxy_systemtests"
+NETWORK_NAME="ccn_proxy_systemtests"
+PROXY_IMAGE="ccn_proxy:devbuild"
+
+echo "Building systemtests image..."
+# we can't assume the build base image exists, but this will almost always be a noop
+docker build -t "ccn_proxy_build_base" -f ./build/Dockerfile.base .
+docker build -t $IMAGE_NAME -f ./build/Dockerfile.systemtests .
+
+function ip_for_container {
+    docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $1
+}
+
+# ----- SETUP -------------------------------------------------------------------
+
+echo "Creating docker network $NETWORK_NAME"
+docker network rm $NETWORK_NAME 2>/dev/null || true
+docker network create $NETWORK_NAME
+
+
+echo "Starting etcd container..."
+ETCD_CONTAINER_NAME="etcd_ccn_proxy_systemtests"
+ETCD_CONTAINER_ID=$(
+    docker run -d \
+       -p 2379:2379 \
+       --name $ETCD_CONTAINER_NAME \
+       --network $NETWORK_NAME \
+       quay.io/coreos/etcd:v2.3.7 \
+       --listen-client-urls http://0.0.0.0:2379 \
+       --advertise-client-urls http://0.0.0.0:2379
+)
+ETCD_CONTAINER_IP=$(ip_for_container $ETCD_CONTAINER_ID)
+echo "etcd running @ $ETCD_CONTAINER_IP:2379"
+
+
+echo "Starting consul container..."
+CONSUL_CONTAINER_NAME="consul_ccn_proxy_systemtests"
+CONSUL_CONTAINER_ID=$(
+    docker run -d \
+	   -p 8500:8500 \
+	   --name $CONSUL_CONTAINER_NAME \
+	   --network $NETWORK_NAME \
+	   consul
+)
+
+CONSUL_CONTAINER_IP=$(ip_for_container $CONSUL_CONTAINER_ID)
+echo "consul running @ $CONSUL_CONTAINER_IP:8500"
+
+
+#
+# NOTE: we start the systemtests container and then later use `docker exec` to
+#       run the tests against etcd and consul.  The reason for starting it like
+#       this is to avoid the chicken-and-egg problem of the systemtests (Go)
+#       code needing to know the IP:port of the proxy and the proxy needing to
+#       know the IP:port of where the MockServer (in the systemtests code) will
+#       be listening.
+#
+echo "Starting systemtests container..."
+PROXY_CONTAINER_NAME="proxy_ccn_proxy_systemtests"
+# run with a tty so that bash (entrypoint) doesn't immediately exit
+SYSTEMTESTS_CONTAINER_ID=$(
+    docker run -d -t \
+        --network $NETWORK_NAME \
+	-e DEBUG="${DEBUG-}" \
+	-e ETCD_CONTAINER_IP="$ETCD_CONTAINER_IP" \
+	-e CONSUL_CONTAINER_IP="$CONSUL_CONTAINER_IP" \
+        ccn_proxy_systemtests
+)
+SYSTEMTESTS_CONTAINER_IP=$(ip_for_container $SYSTEMTESTS_CONTAINER_ID)
+echo "systemtests container running @ $SYSTEMTESTS_CONTAINER_IP"
+
+
+echo "Starting etcd proxy container..."
+ETCD_PROXY_CONTAINER_ID=$(
+    docker run -d \
+	   -p 10000:10000 \
+	   -v $(pwd)/local_certs:/local_certs:ro \
+	   --network $NETWORK_NAME \
+	   $PROXY_IMAGE \
+	   --data-store-address="etcd://$ETCD_CONTAINER_IP:2379" \
+	   --tls-certificate=/local_certs/cert.pem \
+	   --tls-key-file=/local_certs/local.key \
+	   --listen-address=0.0.0.0:10000 \
+	   --netmaster-address="$SYSTEMTESTS_CONTAINER_IP:9999"
+)
+ETCD_PROXY_CONTAINER_IP=$(ip_for_container $ETCD_PROXY_CONTAINER_ID)
+ETCD_PROXY_ADDRESS="$ETCD_PROXY_CONTAINER_IP:10000"
+echo "etcd proxy container running @ $ETCD_PROXY_CONTAINER_IP:10000"
+
+
+echo "Starting consul proxy container..."
+CONSUL_PROXY_CONTAINER_ID=$(
+    docker run -d \
+	   -p 10001:10001 \
+	   -v $(pwd)/local_certs:/local_certs:ro \
+	   --network $NETWORK_NAME \
+	   $PROXY_IMAGE \
+	   --data-store-address="consul://$CONSUL_CONTAINER_IP:8500" \
+	   --tls-certificate=/local_certs/cert.pem \
+	   --tls-key-file=/local_certs/local.key \
+	   --listen-address=0.0.0.0:10001 \
+	   --netmaster-address="$SYSTEMTESTS_CONTAINER_IP:9999"
+)
+CONSUL_PROXY_CONTAINER_IP=$(ip_for_container $CONSUL_PROXY_CONTAINER_ID)
+CONSUL_PROXY_ADDRESS="$CONSUL_PROXY_CONTAINER_IP:10001"
+echo "consul proxy container running @ $CONSUL_PROXY_CONTAINER_IP:10001"
+
+# ----- TEST EXECUTION ----------------------------------------------------------
+
+echo "Executing systemtests..."
+# you can't pass in envvars to docker exec and we didn't know the IPs of the proxy
+# containers when we started this container, so pass them in as arguments here.
+docker exec $SYSTEMTESTS_CONTAINER_ID bash ./scripts/systemtests_in_container.sh $ETCD_PROXY_ADDRESS $CONSUL_PROXY_ADDRESS || true
+
+# ---- CLEANUP ------------------------------------------------------------------
+
+echo "Stopping etcd proxy container..."
+docker rm -f -v $ETCD_PROXY_CONTAINER_ID
+
+echo "Stopping consul proxy container..."
+docker rm -f -v $CONSUL_PROXY_CONTAINER_ID
+
+echo "Shutting down systemtests container..."
+docker rm -f -v $SYSTEMTESTS_CONTAINER_ID
+
+echo "Shutting down etcd container..."
+docker rm -f -v $ETCD_CONTAINER_NAME
+
+echo "Shutting down consul container..."
+docker rm -f -v $CONSUL_CONTAINER_NAME
+
+echo "Destroying docker network $NETWORK_NAME"
+docker network rm $NETWORK_NAME
diff --git a/scripts/systemtests_in_container.sh b/scripts/systemtests_in_container.sh
new file mode 100755
index 00000000..54234013
--- /dev/null
+++ b/scripts/systemtests_in_container.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+echo ""
+echo "Running systemtests against etcd"
+echo ""
+
+set -x
+PROXY_ADDRESS=$1 USE_DATASTORE_ADDRESS="etcd://$ETCD_CONTAINER_IP:2379" go test -v -timeout 5m ./systemtests -check.v
+set +x
+
+echo ""
+echo "Running systemtests against consul"
+echo ""
+
+set -x
+PROXY_ADDRESS=$2 USE_DATASTORE_ADDRESS="consul://$CONSUL_CONTAINER_IP:8500" go test -v -timeout 5m ./systemtests -check.v
+set +x
diff --git a/systemtests/auth_test.go b/systemtests/auth_test.go
index 1c7c72c3..1509166e 100644
--- a/systemtests/auth_test.go
+++ b/systemtests/auth_test.go
@@ -13,7 +13,7 @@ import (
 func (s *systemtestSuite) TestAdminAuthorizationResponse(c *C) {
 	s.addUser(c, username)
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		// grant admin access to username, tenant name is redundant and should be ignored
 		data := `{"PrincipalName":"` + username + `","local":true,"role":"` + types.Admin.String() + `","tenantName":"XXX"}`
 		authz := s.addAuthorization(c, data, adToken)
@@ -44,7 +44,7 @@ func (s *systemtestSuite) TestAdminAuthorizationResponse(c *C) {
 func (s *systemtestSuite) TestTenantAuthorizationResponse(c *C) {
 	s.addUser(c, username)
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		// grant ops access to username, tenant name is required and should not be ignored
 		data := `{"PrincipalName":"` + username + `","local":true,"role":"` + types.Ops.String() + `","tenantName":""}`
 		endpoint := "/api/v1/ccn_proxy/authorizations"
diff --git a/systemtests/basic_test.go b/systemtests/basic_test.go
index a8701621..995ba3b7 100644
--- a/systemtests/basic_test.go
+++ b/systemtests/basic_test.go
@@ -4,7 +4,6 @@ import (
 	"io/ioutil"
 	"net/http"
 
-	"github.com/contiv/ccn_proxy/proxy"
 	. "gopkg.in/check.v1"
 )
 
@@ -22,9 +21,7 @@ const (
 // TestLogin tests that valid credentials successfully authenticate and that
 // invalid credentials result in failed authentication.
 func (s *systemtestSuite) TestLogin(c *C) {
-
-	// test local user login
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		//
 		// valid credentials
 		//
@@ -43,7 +40,7 @@ func (s *systemtestSuite) TestLogin(c *C) {
 	})
 
 	// test LDAP user login
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		adToken := adminToken(c)
 
 		// add LDAP configuration
@@ -77,7 +74,7 @@ func (s *systemtestSuite) TestLogin(c *C) {
 // TestRequestProxying tests that authenticated requests are proxied to the mock
 // server and the response is returned properly.
 func (s *systemtestSuite) TestRequestProxying(c *C) {
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		data := `{"foo":"bar"}`
 		endpoint := "/api/v1/networks/"
@@ -110,7 +107,7 @@ func (s *systemtestSuite) TestRequestProxying(c *C) {
 // TestPOSTBody tests that the body of a POST request is received by the mock
 // server exactly as we sent it to the proxy.
 func (s *systemtestSuite) TestPOSTBody(c *C) {
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		data := `{"foo":"bar"}`
 		endpoint := "/api/v1/networks/foo/"
diff --git a/systemtests/init_test.go b/systemtests/init_test.go
index ea080e92..0bb478d3 100644
--- a/systemtests/init_test.go
+++ b/systemtests/init_test.go
@@ -7,15 +7,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/contiv/ccn_proxy/auth"
-	"github.com/contiv/ccn_proxy/common"
 	"github.com/contiv/ccn_proxy/common/test"
 	"github.com/contiv/ccn_proxy/common/types"
-	"github.com/contiv/ccn_proxy/db"
 	"github.com/contiv/ccn_proxy/proxy"
 	"github.com/contiv/ccn_proxy/state"
 
@@ -30,35 +27,38 @@ var (
 
 	opsUsername = types.Ops.String()
 	opsPassword = types.Ops.String()
+
+	proxyHost = ""
 )
 
+// Test is the entrypoint for the systemtests suite.
+// depending on the value of the USE_DATASTORE envvar, the tests will either run
+// against etcd or consul.  the datastore is assumed to be fresh and with no
+// existing state.
 func Test(t *testing.T) {
 	if len(os.Getenv("DEBUG")) > 0 {
 		log.SetLevel(log.DebugLevel)
 	}
 
-	datastore := test.GetDatastore()
+	// PROXY_ADDRESS is set in ./scripts/systemtests_in_container.sh
+	proxyHost = os.Getenv("PROXY_ADDRESS")
+	if 0 == len(proxyHost) {
+		panic("you must supply a PROXY_ADDRESS (e.g., 1.2.3.4:12345)")
+	}
+
 	datastoreAddress := test.GetDatastoreAddress()
 
+	log.Info("Initializing datastore")
 	if err := state.InitializeStateDriver(datastoreAddress); err != nil {
 		log.Fatalln(err)
 	}
 
-	// set `tls_key_file` in Globals
-	exec.Command("/bin/sh", "-c", "make generate-certificate")
-	common.Global().Set("tls_key_file", "../local_certs/local.key")
-
-	// cleanup users and principals
-	test.CleanupDatastore(datastore, []string{
-		db.GetPath(db.RootLocalUsers),
-		db.GetPath(db.RootLdapConfiguration),
-		db.GetPath("authorizations"),
-	})
-
+	log.Info("Adding default users")
 	if err := auth.AddDefaultUsers(); err != nil {
 		log.Fatalln(err)
 	}
 
+	// execute the systemtests
 	TestingT(t)
 }
 
@@ -66,49 +66,25 @@ type systemtestSuite struct{}
 
 var _ = Suite(&systemtestSuite{})
 
-// ===== TEST PROXY SERVER ======================================================
-
-const proxyHost = "localhost:50000"
-const netmasterHost = "localhost:60000"
-
-// NewTestProxy returns a running proxy server
-func NewTestProxy() *proxy.Server {
-	p := proxy.NewServer(&proxy.Config{
-		Name:    "CCN Proxy",
-		Version: "systemtests",
-
-		// NOTE: instead of hardcoding the two ports here, we could just
-		//       randomly bind to ports until we find two available ones.
-		//       we can cross that bridge if it's worth it.
-		NetmasterAddress: netmasterHost,
-		ListenAddress:    proxyHost,
-
-		// cert and key path are relative to /systemtests and since
-		// `generate-certificates` is a make dependency, we can assume they exist
-		TLSCertificate: "../local_certs/cert.pem",
-		TLSKeyFile:     "../local_certs/local.key",
-	})
-	p.DisableKeepalives()
-	go p.Serve()
-
-	return p
-}
-
 // ===== MISC FUNCTIONS =========================================================
 
-// runTest is a convenience function which encapsulates the logic of creating an
-// instance of proxy.Server and setting its upstream to an instance of MockServer.
-// running each test inside of a runTest() call guarantees that no routes or
-// shenanigans from any previous tests are present.
-func runTest(f func(*proxy.Server, *MockServer)) {
-	p := NewTestProxy()
+// runTest is a convenience function which calls the passed in function and
+// gives it a programmable MockServer as an argument.
+// the ccn_proxy:devbuild container which is started before any tests run is
+// configured to use the MockServer as its "netmaster".
+// see basic_test.go for some examples of how to use it.
+func runTest(f func(*MockServer)) {
 	ms := NewMockServer()
 
-	// TODO: there is a race condition here regarding the goroutines where
-	//       http.Serve() is eventually called on the http.Server objects
-	//       created by NewTestProxy() and NewMockServer() above. Serve()
-	//       does not provide any notification mechanism for when it's ready
-	//       and it blocks when called, so there will be a very short window
+	// there is, however, no race condition on shutdown.  this blocks until the
+	// listener is destroyed.
+	defer ms.Stop()
+
+	// TODO: there is a race condition here regarding the goroutine where
+	//       http.Serve() is eventually called on the http.Server object
+	//       created by NewMockServer(). Serve() does not provide any
+	//       notification mechanism for when its listener is ready and it
+	//       blocks when called, so there will be a very short window
 	//       between us starting the proxy and mock servers and them actually
 	//       being available to handle requests.
 	//
@@ -118,12 +94,9 @@ func runTest(f func(*proxy.Server, *MockServer)) {
 	//       We could send HTTP requests in a loop until one succeeds on
 	//       each server or something, but this is an acceptable stopgap
 	//       for now.
-	time.Sleep(25 * time.Millisecond)
-
-	f(p, ms)
+	time.Sleep(100 * time.Millisecond)
 
-	ms.Stop()
-	p.Stop()
+	f(ms)
 }
 
 // login returns the user's token or returns an error if authentication fails.
diff --git a/systemtests/local_user_test.go b/systemtests/local_user_test.go
index 95a299a5..6c1584dc 100644
--- a/systemtests/local_user_test.go
+++ b/systemtests/local_user_test.go
@@ -2,7 +2,6 @@ package systemtests
 
 import (
 	"github.com/contiv/ccn_proxy/common/types"
-	"github.com/contiv/ccn_proxy/proxy"
 	. "gopkg.in/check.v1"
 )
 
@@ -13,7 +12,7 @@ var (
 
 // TestBuiltinLocalUsers tests that builtInUsers are pre-defined in the system
 func (s *systemtestSuite) TestBuiltinLocalUsers(c *C) {
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		for _, username := range builtInUsers {
 			loginAs(c, username, username)
 		}
@@ -24,7 +23,7 @@ func (s *systemtestSuite) TestBuiltinLocalUsers(c *C) {
 // TestLocalUserEndpoints tests ccn_proxy's local user endpoints
 func (s *systemtestSuite) TestLocalUserEndpoints(c *C) {
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		token := adminToken(c)
 
 		for _, username := range newUsers {
@@ -71,7 +70,7 @@ func (s *systemtestSuite) TestLocalUserUpdateEndpoint(c *C) {
 // userUpdateEndpoint tests update on local user
 func (s *systemtestSuite) userUpdate(c *C) {
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		token := adminToken(c)
 
 		for _, username := range newUsers {
@@ -110,7 +109,7 @@ func (s *systemtestSuite) userUpdate(c *C) {
 // builtInUserUpdate tests built-in user update functionality
 func (s *systemtestSuite) builtInUserUpdate(c *C) {
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		token := adminToken(c)
 
 		for _, username := range builtInUsers {
@@ -147,7 +146,7 @@ func (s *systemtestSuite) builtInUserUpdate(c *C) {
 // TestLocalUserDeleteEndpoint tests ccn_proxy's local user delete endpoint
 func (s *systemtestSuite) TestLocalUserDeleteEndpoint(c *C) {
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		token := adminToken(c)
 
 		// add and delete new users
diff --git a/systemtests/mock_server.go b/systemtests/mock_server.go
index 5384027e..833fc743 100644
--- a/systemtests/mock_server.go
+++ b/systemtests/mock_server.go
@@ -53,7 +53,7 @@ func (ms *MockServer) AddHandler(path string, f func(http.ResponseWriter, *http.
 func (ms *MockServer) Serve() {
 	var err error
 
-	ms.listener, err = net.Listen("tcp", netmasterHost)
+	ms.listener, err = net.Listen("tcp", "0.0.0.0:9999")
 	if err != nil {
 		log.Fatal("net.Listen: ", err)
 		return
diff --git a/systemtests/rbac_test.go b/systemtests/rbac_test.go
index 94cf8fd6..cbd3e0bd 100644
--- a/systemtests/rbac_test.go
+++ b/systemtests/rbac_test.go
@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/contiv/ccn_proxy/proxy"
 	. "gopkg.in/check.v1"
 )
 
@@ -48,7 +47,7 @@ func (s *systemtestSuite) TestRBACFilters(c *C) {
 	s.addUser(c, username)
 
 	// test all list endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		copy := []string{"tenants"}
 		for resource := range epSuffixes {
@@ -105,7 +104,7 @@ func (s *systemtestSuite) TestRBACFilters(c *C) {
 	})
 
 	// test adminOnly list endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		for resource := range adminEpSuffixes {
 			endpoint := "/api/v1/" + resource + "/"
@@ -131,7 +130,7 @@ func (s *systemtestSuite) TestRBACOnDELETERequest(c *C) {
 	s.addUser(c, username)
 
 	// test `tenant` delete
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		endpoint := "/api/v1/tenants/" + tenantName + "/"
 		respData := `{"foo":"bar"}`
@@ -161,7 +160,7 @@ func (s *systemtestSuite) TestRBACOnDELETERequest(c *C) {
 	})
 
 	// test all other endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		for resource, rName := range epSuffixes {
 			endpoint := "/api/v1/" + resource + "/" + rName + "/"
@@ -192,7 +191,7 @@ func (s *systemtestSuite) TestRBACOnDELETERequest(c *C) {
 	})
 
 	// test adminOnly list endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		for resource, rName := range adminEpSuffixes {
 			endpoint := "/api/v1/" + resource + "/" + rName + "/"
@@ -219,7 +218,7 @@ func (s *systemtestSuite) TestRBACOnGETRequest(c *C) {
 	s.addUser(c, username)
 
 	// test `tenant` get
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		endpoint := "/api/v1/tenants/" + tenantName + "/"
 		respData := `{"foo":"bar"}`
@@ -256,7 +255,7 @@ func (s *systemtestSuite) TestRBACOnGETRequest(c *C) {
 	})
 
 	// test all other netmaster endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		for resource, rName := range epSuffixes {
 			endpoint := "/api/v1/" + resource + "/" + rName + "/"
@@ -286,7 +285,7 @@ func (s *systemtestSuite) TestRBACOnGETRequest(c *C) {
 	})
 
 	// test adminOnly list endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		for resource, rName := range adminEpSuffixes {
 			endpoint := "/api/v1/" + resource + "/" + rName + "/"
@@ -312,7 +311,7 @@ func (s *systemtestSuite) TestRBACOnPOSTRequest(c *C) {
 	s.addUser(c, username)
 
 	// test `tenant` creation
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		endpoint := "/api/v1/tenants/" + tenantName + "/"
 		tenantCreateReq := `{"tenantName": "` + tenantName + `"}`
@@ -339,7 +338,7 @@ func (s *systemtestSuite) TestRBACOnPOSTRequest(c *C) {
 	})
 
 	// test all other netmaster endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		for resource, rName := range epSuffixes {
 			endpoint := "/api/v1/" + resource + "/" + rName + "/"
 			createReq := `{"tenantName": "` + tenantName + `"}`
@@ -380,7 +379,7 @@ func (s *systemtestSuite) TestRBACOnPOSTRequest(c *C) {
 	})
 
 	// test adminOnly list endpoints
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		for resource, rName := range adminEpSuffixes {
 			endpoint := "/api/v1/" + resource + "/" + rName + "/"
@@ -405,7 +404,7 @@ func (s *systemtestSuite) TestRBACOnPOSTRequest(c *C) {
 // addUser helper function that adds a new local user to the system
 func (s *systemtestSuite) addUser(c *C, username string) {
 	// add new local user
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 		adToken = adminToken(c)
 
 		endpoint := "/api/v1/ccn_proxy/local_users/" + username
@@ -474,7 +473,7 @@ func (s *systemtestSuite) TestAdminRoleRequired(c *C) {
 	// This also sets up the adToken variable
 	s.addUser(c, username)
 
-	runTest(func(p *proxy.Server, ms *MockServer) {
+	runTest(func(ms *MockServer) {
 
 		// login as username, should succeed
 		testuserToken := loginAs(c, username, username)

From b71dbac69665af4c16f5b383fcf83c98b8c2ffa5 Mon Sep 17 00:00:00 2001
From: Bill Robinson <dseevr@users.noreply.github.com>
Date: Wed, 4 Jan 2017 17:57:32 -0800
Subject: [PATCH 2/2] `make checks` now runs in a container

Signed-off-by: Bill Robinson <dseevr@users.noreply.github.com>
---
 build/Dockerfile.checks        |  9 ++++
 scripts/checks.sh              | 91 ++++------------------------------
 scripts/checks_in_container.sh | 82 ++++++++++++++++++++++++++++++
 3 files changed, 101 insertions(+), 81 deletions(-)
 create mode 100644 build/Dockerfile.checks
 create mode 100755 scripts/checks_in_container.sh

diff --git a/build/Dockerfile.checks b/build/Dockerfile.checks
new file mode 100644
index 00000000..3c38c68e
--- /dev/null
+++ b/build/Dockerfile.checks
@@ -0,0 +1,9 @@
+FROM ccn_proxy_build_base
+
+RUN go get github.com/gordonklaus/ineffassign
+RUN go get github.com/golang/lint/golint
+RUN go get github.com/fzipp/gocyclo
+RUN go get github.com/client9/misspell/...
+RUN go get github.com/remyoudompheng/go-misc/deadcode/...
+
+ENTRYPOINT ["/bin/bash", "./scripts/checks_in_container.sh"]
diff --git a/scripts/checks.sh b/scripts/checks.sh
index 6d6bf756..5e81d49d 100755
--- a/scripts/checks.sh
+++ b/scripts/checks.sh
@@ -1,85 +1,14 @@
-#!/bin/sh
+#!/bin/bash
 
-set -e
+set -euo pipefail
 
-dirs=$(go list ./... | sed -e 's!github.com/contiv/ccn_proxy!.!g' | grep -v ./vendor)
-files=$(find . -type f -name '*.go' | grep -v ./vendor)
+CHECKS_CONTAINER_NAME="ccn_proxy_checks"
 
-echo "Running gofmt..."
-set +e
-out=$(gofmt -s -l ${files})
-set -e
-# gofmt can include empty lines in its output
-if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
-then
-  echo 1>&2 "gofmt errors in:"
-  echo 1>&2 "${out}"
-  exit 1
-fi
+docker build -t "ccn_proxy_build_base" -f ./build/Dockerfile.base .
+docker build -t $CHECKS_CONTAINER_NAME -f ./build/Dockerfile.checks .
 
-echo "Running ineffassign..."
-[ -n "`which ineffassign`" ] || go get github.com/gordonklaus/ineffassign
-for i in ${dirs}
-do
-  ineffassign $i
-done
-
-echo "Running golint..."
-[ -n "`which golint`" ] || go get github.com/golang/lint/golint
-set +e
-out=$(golint ./... | grep -vE '^vendor')
-set -e
-if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
-then
-  echo 1>&2 "golint errors in:"
-  echo 1>&2 "${out}"
-  exit 1
-fi
-
-echo "Running govet..."
-set +e
-out=$(go tool vet -composites=false ${dirs} 2>&1 | grep -v vendor)
-set -e
-
-if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
-then
-  echo 1>&2 "go vet errors in:"
-  echo 1>&2 "${out}"
-  exit 1
-fi
-
-echo "Running gocyclo..."
-[ -n "`which gocyclo`" ] || go get github.com/fzipp/gocyclo
-set +e
-out=$(gocyclo -over 15 . | grep -v vendor)
-set -e
-if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
-then
-  echo 1>&2 "gocycle errors in:"
-  echo 1>&2 "${out}"
-  exit 1
-fi
-
-echo "Running misspell..."
-[ -n "`which misspell`" ] || go get github.com/client9/misspell/...
-set +e
-out=$(misspell -locale US -error -i exportfs ${files})
-set -e
-if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
-then
-  echo 1>&2 "misspell errors in:"
-  echo 1>&2 "${out}"
-  exit 1
-fi
-
-echo "Running deadcode..."
-[ -n "`which deadcode`" ] || go get github.com/remyoudompheng/go-misc/deadcode/...
-set +e
-out=$(deadcode ${dirs} 2>&1)
-set -e
-if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
-then
-  echo 1>&2 "deadcode errors in:"
-  echo 1>&2 "${out}"
-  exit 1
-fi
+# the ccn_proxy directory exists in the base image, but it's empty so we'll
+# bindmount the current directory into it.
+docker run --rm \
+       -v $(pwd):/go/src/github.com/contiv/ccn_proxy:ro \
+       $CHECKS_CONTAINER_NAME
diff --git a/scripts/checks_in_container.sh b/scripts/checks_in_container.sh
new file mode 100755
index 00000000..9b18417c
--- /dev/null
+++ b/scripts/checks_in_container.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -e
+
+export PATH="/go/bin:$PATH"
+
+dirs=$(go list ./... | sed -e 's!github.com/contiv/ccn_proxy!.!g' | grep -v ./vendor)
+files=$(find . -type f -name '*.go' | grep -v ./vendor)
+
+echo "Running gofmt..."
+set +e
+out=$(gofmt -s -l ${files})
+set -e
+# gofmt can include empty lines in its output
+if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
+then
+  echo 1>&2 "gofmt errors in:"
+  echo 1>&2 "${out}"
+  exit 1
+fi
+
+echo "Running ineffassign..."
+for i in ${dirs}
+do
+  ineffassign $i
+done
+
+echo "Running golint..."
+set +e
+out=$(golint ./... | grep -vE '^vendor')
+set -e
+if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
+then
+  echo 1>&2 "golint errors in:"
+  echo 1>&2 "${out}"
+  exit 1
+fi
+
+echo "Running govet..."
+set +e
+out=$(go tool vet -composites=false ${dirs} 2>&1 | grep -v vendor)
+set -e
+
+if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
+then
+  echo 1>&2 "go vet errors in:"
+  echo 1>&2 "${out}"
+  exit 1
+fi
+
+echo "Running gocyclo..."
+set +e
+out=$(gocyclo -over 15 . | grep -v vendor)
+set -e
+if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
+then
+  echo 1>&2 "gocyclo errors in:"
+  echo 1>&2 "${out}"
+  exit 1
+fi
+
+echo "Running misspell..."
+set +e
+out=$(misspell -locale US -error -i exportfs ${files})
+set -e
+if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
+then
+  echo 1>&2 "misspell errors in:"
+  echo 1>&2 "${out}"
+  exit 1
+fi
+
+echo "Running deadcode..."
+set +e
+out=$(deadcode ${dirs} 2>&1)
+set -e
+if [ "`echo \"${out}\" | sed '/^$/d' | wc -l`" -gt 0 ]
+then
+  echo 1>&2 "deadcode errors in:"
+  echo 1>&2 "${out}"
+  exit 1
+fi