diff --git a/.gitignore b/.gitignore index a1118e1b1f..eb7a84d1bd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ tmp build/local-includes/* !build/local-includes/README.md /release +debug.test diff --git a/build/Makefile b/build/Makefile index 9ebedb4381..2d77b30044 100644 --- a/build/Makefile +++ b/build/Makefile @@ -46,6 +46,9 @@ GCP_CLUSTER_ZONE ?= us-west1-c # the profile to use when developing on minikube MINIKUBE_PROFILE ?= agones +# Game Server image to use while doing end-to-end tests +GS_TEST_IMAGE ?= gcr.io/agones-images/cpp-simple-server:0.2 + # Directory that this Makefile is in. mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) build_path := $(dir $(mkfile_path)) @@ -63,7 +66,7 @@ controller_tag = $(REGISTRY)/agones-controller:$(VERSION) sidecar_tag = $(REGISTRY)/agones-sdk:$(VERSION) go_version_flags = -ldflags "-X agones.dev/agones/pkg.Version=$(VERSION)" - +DOCKER_RUN ?= docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) # ___ ____ ___ _ _ # / _ \/ ___| |_ _|_ __ ___| |_ _ __| | ___ # | | | \___ \ | || '_ \ / __| | | | |/ _` |/ _ \ @@ -85,6 +88,10 @@ include ./includes/$(osinclude) # personal includes, excluded from the git repository -include ./local-includes/*.mk +ifdef DOCKER_RUN + ensure-build-image += ensure-build-image +endif + # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ # | |/ _` | '__/ _` |/ _ \ __/ __| @@ -102,11 +109,19 @@ build-images: build-controller-image build-agones-sdk-image build-sdks: build-sdk-cpp # Run all tests -test: ensure-build-image test-go test-install-yaml +test: $(ensure-build-image) test-go test-install-yaml # Run go tests test-go: - docker run --rm $(common_mounts) $(build_tag) go test -race $(agones_package)/... + docker run --rm $(common_mounts) $(build_tag) go test -race $(agones_package)/pkg/... \ + $(agones_package)/sdks/... + +# Runs end-to-end tests on the current configured cluster +# For minikube user the minikube-test-e2e targets +test-e2e: + $(DOCKER_RUN) go test -v $(agones_package)/test/e2e/... \ + --kubeconfig /root/.kube/config \ + --gameserver-image=$(GS_TEST_IMAGE) # Run test on install yaml - make sure there is no change # mostly this is for CI @@ -124,34 +139,34 @@ push: push-controller-image push-agones-sdk-image # Installs the current development version of Agones into the Kubernetes cluster install: ALWAYS_PULL_SIDECAR := true install: IMAGE_PULL_POLICY := "Always" -install: ensure-build-image - docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \ - helm upgrade --install --namespace=agones-system \ +install: $(ensure-build-image) + $(DOCKER_RUN) \ + helm upgrade --install --recreate-pods --wait --namespace=agones-system \ --set agones.image.tag=$(VERSION),agones.image.registry=$(REGISTRY),agones.image.controller.pullPolicy=$(IMAGE_PULL_POLICY),agones.image.sdk.alwaysPull=$(ALWAYS_PULL_SIDECAR) \ agones $(mount_path)/install/helm/agones/ # Build a static binary for the gameserver controller -build-controller-binary: ensure-build-image +build-controller-binary: $(ensure-build-image) docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \ -o $(mount_path)/cmd/controller/bin/controller -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/controller # Lint the go source code. # use LINT_TIMEOUT to manipulate the linter timeout lint: LINT_TIMEOUT ?= 15m -lint: ensure-build-image +lint: $(ensure-build-image) docker run --rm $(common_mounts) -w $(mount_path) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \ "/root/gen-lint-exclude.sh && gometalinter --config .exclude.gometalinter.json --deadline=$(LINT_TIMEOUT) -t --skip vendor ./..." # Build the image for the gameserver controller -build-controller-image: ensure-build-image build-controller-binary +build-controller-image: $(ensure-build-image) build-controller-binary docker build $(agones_path)/cmd/controller/ --tag=$(controller_tag) $(DOCKER_BUILD_ARGS) # push the gameservers controller image -push-controller-image: ensure-build-image +push-controller-image: $(ensure-build-image) docker push $(controller_tag) # build the static binary for the gamesever sidecar -build-agones-sdk-binary: ensure-build-image +build-agones-sdk-binary: $(ensure-build-image) docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \ -o $(mount_path)/cmd/sdk-server/bin/sdk-server.linux.amd64 -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/sdk-server docker run --rm -e "GOOS=darwin" -e "GOARCH=amd64" $(common_mounts) $(build_tag) go build \ @@ -162,38 +177,38 @@ build-agones-sdk-binary: ensure-build-image agonessdk-server-$(VERSION).zip sdk-server.darwin.amd64 sdk-server.linux.amd64 sdk-server.windows.amd64.exe # Build the image for the gameserver sidecar -build-agones-sdk-image: ensure-build-image build-agones-sdk-binary +build-agones-sdk-image: $(ensure-build-image) build-agones-sdk-binary docker build $(agones_path)/cmd/sdk-server/ --tag=$(sidecar_tag) $(DOCKER_BUILD_ARGS) # Build the cpp sdk linux archive -build-sdk-cpp: ensure-build-image +build-sdk-cpp: $(ensure-build-image) docker run --rm $(common_mounts) -w $(mount_path)/sdks/cpp $(build_tag) make build install archive VERSION=$(VERSION) # push the gameservers sidecar image -push-agones-sdk-image: ensure-build-image +push-agones-sdk-image: $(ensure-build-image) docker push $(sidecar_tag) # Generate the static install script -gen-install: ensure-build-image +gen-install: $(ensure-build-image) docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \ 'helm template --name=agones-manual --namespace agones-system $(mount_path)/install/helm/agones \ --set agones.controller.generateTLS=false \ > $(mount_path)/install/yaml/install.yaml' # Generate the SDK gRPC server and client code -gen-gameservers-sdk-grpc: ensure-build-image +gen-gameservers-sdk-grpc: $(ensure-build-image) docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-go.sh docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-cpp.sh docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-rust.sh # Generate the client for our CustomResourceDefinition -gen-crd-client: ensure-build-image +gen-crd-client: $(ensure-build-image) docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-crd-client.sh docker run --rm $(common_mounts) -w $(mount_path)/pkg $(build_tag) goimports -w . # Run a bash shell with the developer tools in it. (Creates the image if it doesn't exist) # Can use DOCKER_RUN_ARGS for extra arguments. -shell: ensure-build-image +shell: $(ensure-build-image) docker run -it --rm \ $(common_mounts) \ -w $(mount_path) \ @@ -286,7 +301,7 @@ gcloud-init: ensure-build-config gcloud-test-cluster: GCP_CLUSTER_LEGACYABAC ?= false gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 -gcloud-test-cluster: ensure-build-image +gcloud-test-cluster: $(ensure-build-image) docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \ deployment-manager deployments create $(GCP_CLUSTER_NAME) \ --properties cluster.zone:$(GCP_CLUSTER_ZONE),cluster.name:$(GCP_CLUSTER_NAME),cluster.nodePool.initialNodeCount:$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT),cluster.nodePool.machineType:$(GCP_CLUSTER_NODEPOOL_MACHINETYPE),cluster.legacyAbac:$(GCP_CLUSTER_LEGACYABAC)\ @@ -295,13 +310,31 @@ gcloud-test-cluster: ensure-build-image docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl apply -f $(mount_path)/build/helm.yaml docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm -clean-gcloud-test-cluster: ensure-build-image +clean-gcloud-test-cluster: $(ensure-build-image) docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \ deployment-manager deployments delete $(GCP_CLUSTER_NAME) - + +# Creates a gcloud cluster for end-to-end +# it installs also a consul cluster to handle build system concurrency using a distributed lock +gcloud-e2e-test-cluster: $(ensure-build-image) + docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \ + deployment-manager deployments create e2e-test-cluster \ + --config=$(mount_path)/build/gke-test-cluster/cluster-e2e.yml + GCP_CLUSTER_NAME=e2e-test-cluster GCP_CLUSTER_ZONE=us-west1-c $(MAKE) gcloud-auth-cluster + docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \ + kubectl apply -f $(mount_path)/build/helm.yaml + docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm --wait && \ + helm install --wait --set Replicas=1,uiService.type=ClusterIP --name consul stable/consul + +# Deletes the gcloud e2e cluster and cleanup any left pvc volumes +clean-gcloud-e2e-test-cluster: $(ensure-build-image) + docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \ + helm delete --purge consul && kubectl delete pvc -l component=consul-consul + GCP_CLUSTER_NAME=e2e-test-cluster $(MAKE) clean-gcloud-test-cluster + # Pulls down authentication information for kubectl against a cluster, name can be specified through GCP_CLUSTER_NAME # (defaults to 'test-cluster') -gcloud-auth-cluster: ensure-build-image +gcloud-auth-cluster: $(ensure-build-image) docker run --rm $(common_mounts) $(build_tag) gcloud config set container/cluster $(GCP_CLUSTER_NAME) docker run --rm $(common_mounts) $(build_tag) gcloud config set compute/zone $(GCP_CLUSTER_ZONE) docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME) @@ -309,7 +342,7 @@ gcloud-auth-cluster: ensure-build-image # authenticate our docker configuration so that you can do a docker push directly # to the gcr.io repository -gcloud-auth-docker: ensure-build-image +gcloud-auth-docker: $(ensure-build-image) docker run --rm $(common_mounts) $(build_tag) gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io # Clean the gcloud configuration @@ -328,7 +361,7 @@ clean-gcloud-config: # # Use MINIKUBE_DRIVER variable to change the VM driver # (defaults virtualbox for Linux and macOS, hyperv for windows) if you so desire. -minikube-test-cluster: ensure-build-image minikube-agones-profile +minikube-test-cluster: $(ensure-build-image) minikube-agones-profile # localkube bootstrapper fixes issues with profiles $(MINIKUBE) start --kubernetes-version v1.10.0 --vm-driver $(MINIKUBE_DRIVER) \ --bootstrapper=localkube \ @@ -355,7 +388,7 @@ minikube-agones-profile: # Connecting to minikube requires so enhanced permissions, so use this target # instead of `make shell` to start an interactive shell for development on minikube. -minikube-shell: ensure-build-image minikube-agones-profile +minikube-shell: $(ensure-build-image) minikube-agones-profile $(MAKE) shell DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount) $(DOCKER_RUN_ARGS)" # Push the local Agones Docker images that have already been built @@ -373,3 +406,7 @@ minikube-install: minikube-agones-profile # Use TAG to specify the image to transfer into minikube minikube-transfer-image: docker save $(TAG) | ($(MINIKUBE_DOCKER_ENV) && docker load) + +# Runs e2e tests against our minikube +minikube-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount) +minikube-test-e2e: minikube-agones-profile test-e2e diff --git a/build/README.md b/build/README.md index 1da41ef855..2b0079842d 100644 --- a/build/README.md +++ b/build/README.md @@ -175,6 +175,8 @@ Now that the images are pushed, to install the development version (with all ima run `make install` and Agones will install the image that you just built and pushed on the test cluster you created at the beginning of this section. (if you want to see the resulting installation yaml, you can find it in `build/.install.yaml`) +Finally to run end-to-end tests against your development version previously installed in your test cluster run `make test-e2e`, this will validate the whole application flow (from start to finish). If you're curious about how they work head to [tests/e2e](../test/e2e/) + ### Running a Test Minikube cluster This will setup a [Minikube](https://github.com/kubernetes/minikube) cluster, running on an `agones` profile, @@ -226,6 +228,8 @@ For example: $ make minikube-transfer-image TAG=myimage:0.1 ``` +Running end-to-end tests on minikube is done via the `make minikube-test-e2e` target. This target use the same `make test-e2e` but also setup some prerequisites for use with a minikube cluster. + ### Next Steps Have a look in the [examples](../examples) folder to see examples of running Game Servers on Agones. @@ -274,6 +278,13 @@ Pushes all built images up to the `$(REGISTRY)` #### `make install` Installs the current development version of Agones into the Kubernetes cluster +### `make test-e2e` +Runs end-to-end tests on the previously installed version of Agones. +These tests validate Agones flow from start to finish. + +It uses the kube config (located by default in `~/.kube`) to target a Kubernetes cluster. +See [`make minikube-test-e2e`](#make-minikube-test-e2e) to run end-to-end tests on minikube. + #### `make shell` Run a bash shell with the developer tools (go tooling, kubectl, etc) and source code in it. @@ -349,6 +360,10 @@ via `make build` or `make build-images` into the "agones" minikube instance. Installs the current development version of Agones into the Kubernetes cluster. Use this instead of `make install`, as it disables PullAlways on the install.yaml +### `make minikube-test-e2e` +Runs end-to-end tests on the previously installed version of Agones. +These tests validate Agones flow from start to finish. + #### `make minikube-shell` Connecting to Minikube requires so enhanced permissions, so use this target instead of `make shell` to start an interactive shell for development on Minikube. diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile index 36d033cfed..823d39ac7b 100644 --- a/build/build-image/Dockerfile +++ b/build/build-image/Dockerfile @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# ForceUpdate 5 -- change here if you need to force a rebuild +# ForceUpdate 6 -- change here if you need to force a rebuild # compiling proto + grpc takes an exceptionally long time # so we'll use a base from `base` - which is manually built using the below tag. FROM gcr.io/agones-images/grpc-cxx:1.12 RUN apt-get update && \ - apt-get install -y wget rsync make python bash-completion zip nano jq && \ + apt-get install -y wget psmisc rsync make python bash-completion zip nano jq && \ apt-get clean # install go @@ -39,17 +39,23 @@ RUN wget -q https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip && un ENV PATH /usr/local/go/bin:/go/bin:/opt/google-cloud-sdk/bin:$PATH # RUN gcloud components update -RUN gcloud components update && gcloud components install kubectl +RUN gcloud components update + +# install kubectl without gcloud as we want the last version +ENV KUBECTL_VER 1.11.0 +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \ + chmod go+rx ./kubectl && \ + mv ./kubectl /usr/local/bin/kubectl RUN echo "source <(kubectl completion bash)" >> /root/.bashrc # install Helm package manager ENV HELM_VER 2.9.1 ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VER}-linux-amd64.tar.gz RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \ - && tar -zxvf /tmp/helm.tar.gz -C /tmp \ - && mv /tmp/linux-amd64/helm /usr/local/bin/helm \ - && chmod go+rx /usr/local/bin/helm \ - && rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64 + && tar -zxvf /tmp/helm.tar.gz -C /tmp \ + && mv /tmp/linux-amd64/helm /usr/local/bin/helm \ + && chmod go+rx /usr/local/bin/helm \ + && rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64 RUN echo "source <(helm completion bash)" >> /root/.bashrc # install go-proto-gen 1.1 diff --git a/build/e2e-image/Dockerfile b/build/e2e-image/Dockerfile new file mode 100644 index 0000000000..de9cdc00ef --- /dev/null +++ b/build/e2e-image/Dockerfile @@ -0,0 +1,52 @@ +FROM gcr.io/cloud-builders/gcloud-slim + +RUN apt-get update && \ + apt-get install -y wget psmisc make python jq zip && \ + apt-get clean + +# install go +WORKDIR /usr/local +ENV GO_VERSION=1.10.3 +ENV GOPATH /go +RUN wget -q https://redirector.gvt1.com/edgedl/go/go${GO_VERSION}.linux-amd64.tar.gz && \ + tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz && mkdir ${GOPATH} + +ENV PATH /usr/local/go/bin:/go/bin:$PATH + +# install kubectl without gcloud as we want the last version +ENV KUBECTL_VER 1.11.0 +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \ + chmod go+rx ./kubectl && \ + mv ./kubectl /usr/local/bin/kubectl + +# install Helm package manager +ENV HELM_VER 2.9.1 +ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VER}-linux-amd64.tar.gz +RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \ + && tar -zxvf /tmp/helm.tar.gz -C /tmp \ + && mv /tmp/linux-amd64/helm /usr/local/bin/helm \ + && chmod go+rx /usr/local/bin/helm \ + && rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64 + +# set up Consul. +ENV CONSUL_VERSION=1.2.1 +ENV HASHICORP_RELEASES=https://releases.hashicorp.com +RUN mkdir -p /tmp/build && \ + wget ${HASHICORP_RELEASES}/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip \ + -P /tmp/build/ && \ + unzip -d /usr/local/bin/ /tmp/build/consul_${CONSUL_VERSION}_linux_amd64.zip && \ + cd /tmp && \ + rm -rf /tmp/build && \ + chmod go+rx /usr/local/bin/consul && \ + # tiny smoke test to ensure the binary we downloaded runs + consul version + +# make sure we keep the path to go +RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc +# scripts +COPY *.sh /root/ +RUN chmod +x /root/*.sh + +WORKDIR /go + +ENTRYPOINT [ "/root/entrypoint.sh" ] \ No newline at end of file diff --git a/build/e2e-image/e2e.sh b/build/e2e-image/e2e.sh new file mode 100755 index 0000000000..06dff13d78 --- /dev/null +++ b/build/e2e-image/e2e.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +echo "installing current release" +DOCKER_RUN= make install +echo "starting e2e test" +DOCKER_RUN= make test-e2e diff --git a/build/e2e-image/entrypoint.sh b/build/e2e-image/entrypoint.sh new file mode 100644 index 0000000000..4b871fec36 --- /dev/null +++ b/build/e2e-image/entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +export SHELL="/bin/bash" +export KUBECONFIG="/root/.kube/config" +mkdir -p /go/src/agones.dev/agones/ /root/.kube/ +cp -r /workspace/. /go/src/agones.dev/agones/ +cd /go/src/agones.dev/agones/build +if [ "$1" = 'local' ] +then + gcloud auth login +fi +gcloud container clusters get-credentials e2e-test-cluster \ + --zone=us-west1-c --project=agones-images +kubectl port-forward statefulset/consul-consul 8500:8500 & +echo "Waiting consul port-forward to launch on 8500..." +timeout 60 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' 127.0.0.1 8500 +echo "consul port-forward launched. Starting e2e tests..." +consul lock -child-exit-code=true -timeout 5m -try 5m -verbose LockE2E /root/e2e.sh +killall -q kubectl + diff --git a/build/gke-test-cluster/cluster-e2e.yml b/build/gke-test-cluster/cluster-e2e.yml new file mode 100644 index 0000000000..f7406c3d1b --- /dev/null +++ b/build/gke-test-cluster/cluster-e2e.yml @@ -0,0 +1,44 @@ +# Copyright 2018 Google Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: +- name: e2e-test-cluster + type: container.v1.cluster + properties: + zone: us-west1-c + cluster: + name: e2e-test-cluster + description: End to end tests cluster for Agones + initialClusterVersion: 1.10.5-gke.0 + nodePools: + - name: "default" + initialNodeCount: 2 + config: + machineType: n1-standard-1 + tags: + - game-server + oauthScopes: + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/cloud-platform +- name: game-server-firewall + type: compute.beta.firewall + properties: + name: game-server + description: "Firewall to allow game server udp traffic" + targetTags: + - "game-server" + allowed: + - IPProtocol: udp + ports: + - "7000-8000" diff --git a/cloudbuild.yaml b/cloudbuild.yaml index aed714f5e5..755d5e39f6 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -38,6 +38,13 @@ steps: args: [ "build", "push" ] # build all the things, and push images waitFor: ['lint'] id: build +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '-f', 'Dockerfile', '-t', 'e2e-runner', '.'] + dir: 'build/e2e-image' + id: build-e2e + waitFor: ["lint"] +- name: 'e2e-runner' + waitFor: ['build','build-e2e'] - name: 'gcr.io/cloud-builders/gsutil' dir: "sdks/cpp/bin" args: ['cp', '*.tar.gz', 'gs://agones-artifacts/cpp-sdk'] diff --git a/install/helm/agones/templates/controller.yaml b/install/helm/agones/templates/controller.yaml index b5b3996a35..a74108694b 100644 --- a/install/helm/agones/templates/controller.yaml +++ b/install/helm/agones/templates/controller.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: agones-controller @@ -24,6 +24,12 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: + selector: + matchLabels: + stable.agones.dev/role: controller + app: {{ template "agones.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} replicas: 1 strategy: type: Recreate @@ -32,7 +38,6 @@ spec: labels: stable.agones.dev/role: controller app: {{ template "agones.name" . }} - chart: {{ template "agones.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml index 0423400b8d..a4df13fc61 100644 --- a/install/yaml/install.yaml +++ b/install/yaml/install.yaml @@ -757,7 +757,7 @@ spec: # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: agones-controller @@ -769,6 +769,12 @@ metadata: release: agones-manual heritage: Tiller spec: + selector: + matchLabels: + stable.agones.dev/role: controller + app: agones + release: agones-manual + heritage: Tiller replicas: 1 strategy: type: Recreate @@ -777,7 +783,6 @@ spec: labels: stable.agones.dev/role: controller app: agones - chart: agones-0.4.0.rc release: agones-manual heritage: Tiller spec: diff --git a/pkg/util/webhooks/webhooks.go b/pkg/util/webhooks/webhooks.go index fea21bc0ec..da37c4092c 100644 --- a/pkg/util/webhooks/webhooks.go +++ b/pkg/util/webhooks/webhooks.go @@ -78,7 +78,7 @@ func NewWebHook(certFile, keyFile string) *WebHook { func (wh *WebHook) Run(workers int, stop <-chan struct{}) error { go func() { <-stop - wh.server.Close() // nolint: errcheck + wh.server.Close() // nolint: errcheck,gosec }() wh.logger.WithField("webook", wh).Infof("https server started") diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000000..a0fe4d0896 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,24 @@ +# E2E Testing + +End-to-end (e2e) testing is automated testing for real user scenarios. + +## Build and run test + +Prerequisites: +- a running k8s cluster (kube config is passed as arguments). +- Have kubeconfig file ready. + +e2e tests are written as Go test. All go test techniques apply, e.g. picking +what to run, timeout length. + +To run e2e tests on your kubectl configured cluster: + +``` +make test-e2e +``` + +To run on minikube use the special target: + +``` +make minikube-test-e2e +``` diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go new file mode 100644 index 0000000000..1f78929e33 --- /dev/null +++ b/test/e2e/framework/framework.go @@ -0,0 +1,134 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package framework is a package helping setting up end-to-end testing accross a +// Kubernetes cluster. +package framework + +import ( + "fmt" + "net" + "time" + + "agones.dev/agones/pkg/apis/stable/v1alpha1" + "agones.dev/agones/pkg/client/clientset/versioned" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + // required to use gcloud login see: https://github.com/kubernetes/client-go/issues/242 + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/client-go/tools/clientcmd" +) + +// Framework is a testing framework +type Framework struct { + KubeClient kubernetes.Interface + AgonesClient versioned.Interface + GameServerImage string +} + +// New setups a testing framework using a kubeconfig path and the game server image to use for testing. +func New(kubeconfig, gsimage string) (*Framework, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, errors.Wrap(err, "build config from flags failed") + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, errors.Wrap(err, "creating new kube-client failed") + } + + agonesClient, err := versioned.NewForConfig(config) + if err != nil { + return nil, errors.Wrap(err, "creating new agones-client failed") + } + + return &Framework{ + KubeClient: kubeClient, + AgonesClient: agonesClient, + GameServerImage: gsimage, + }, nil +} + +// CreateGameServerAndWaitUntilReady Creates a GameServer and wait for its state to become ready. +func (f *Framework) CreateGameServerAndWaitUntilReady(ns string, gs *v1alpha1.GameServer) (*v1alpha1.GameServer, error) { + newGs, err := f.AgonesClient.StableV1alpha1().GameServers(ns).Create(gs) + if err != nil { + return nil, fmt.Errorf("creating %v GameServer instances failed (%v): %v", gs.Spec, gs.Name, err) + } + + readyGs, err := f.WaitForGameServerState(newGs, v1alpha1.Ready, 5*time.Minute) + + if err != nil { + return nil, fmt.Errorf("waiting for %v GameServer instance readiness timed out (%v): %v", + gs.Spec, gs.Name, err) + } + + return readyGs, nil +} + +// WaitForGameServerState Waits untils the gameserver reach a given state before the timeout expires +func (f *Framework) WaitForGameServerState(gs *v1alpha1.GameServer, state v1alpha1.State, + timeout time.Duration) (*v1alpha1.GameServer, error) { + var pollErr error + var readyGs *v1alpha1.GameServer + + err := wait.PollImmediate(2*time.Second, timeout, func() (bool, error) { + readyGs, pollErr = f.AgonesClient.StableV1alpha1().GameServers(gs.Namespace).Get(gs.Name, v1.GetOptions{}) + + if pollErr != nil { + return false, nil + } + + if readyGs.Status.State == state { + return true, nil + } + + return false, nil + }) + if err != nil { + return nil, errors.Wrapf(pollErr, "waiting for GameServer to be %v %v/%v: %v", + state, gs.Namespace, gs.Name, err) + } + return readyGs, nil +} + +// CleanUp Delete all agones resources in a given namespace +func (f *Framework) CleanUp(ns string) error { + return f.AgonesClient.StableV1alpha1().GameServers(ns). + DeleteCollection(&v1.DeleteOptions{}, v1.ListOptions{}) +} + +// PingGameServer pings a gameserver and returns its reply +func PingGameServer(msg, address string) (reply string, err error) { + conn, err := net.Dial("udp", address) + if err != nil { + return "", err + } + defer func() { + err = conn.Close() + }() + _, err = conn.Write([]byte(msg)) + if err != nil { + return "", errors.Wrapf(err, "Could not write message %s", msg) + } + b := make([]byte, 1024) + n, err := conn.Read(b) + if err != nil { + return "", err + } + return string(b[:n]), nil +} diff --git a/test/e2e/gameserver_test.go b/test/e2e/gameserver_test.go new file mode 100644 index 0000000000..12d5171193 --- /dev/null +++ b/test/e2e/gameserver_test.go @@ -0,0 +1,69 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "testing" + + "agones.dev/agones/pkg/apis/stable/v1alpha1" + e2eframework "agones.dev/agones/test/e2e/framework" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const defaultNs = "default" + +func TestCreateConnect(t *testing.T) { + t.Parallel() + gs := &v1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{GenerateName: "udp-server", Namespace: defaultNs}, + Spec: v1alpha1.GameServerSpec{ + Container: "udp-server", + Ports: []v1alpha1.GameServerPort{v1alpha1.GameServerPort{ + ContainerPort: 7654, + Name: "gameport", + PortPolicy: v1alpha1.Dynamic, + Protocol: corev1.ProtocolUDP, + }}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "udp-server", + Image: framework.GameServerImage}}, + }, + }, + }, + } + readyGs, err := framework.CreateGameServerAndWaitUntilReady(defaultNs, gs) + + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, len(readyGs.Status.Ports), 1) + assert.NotEmpty(t, readyGs.Status.Ports[0].Port) + assert.NotEmpty(t, readyGs.Status.Address) + assert.NotEmpty(t, readyGs.Status.NodeName) + assert.Equal(t, readyGs.Status.State, v1alpha1.Ready) + + reply, err := e2eframework.PingGameServer("Hello World !", fmt.Sprintf("%s:%d", readyGs.Status.Address, + readyGs.Status.Ports[0].Port)) + + if err != nil { + t.Fatalf("Could ping GameServer: %v", err) + } + + assert.Equal(t, reply, "ACK: Hello World !\n") +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go new file mode 100644 index 0000000000..df04334bd1 --- /dev/null +++ b/test/e2e/main_test.go @@ -0,0 +1,57 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "log" + "os" + "os/user" + "path/filepath" + "testing" + + e2eframework "agones.dev/agones/test/e2e/framework" +) + +var framework *e2eframework.Framework + +func TestMain(m *testing.M) { + usr, _ := user.Current() + kubeconfig := flag.String("kubeconfig", filepath.Join(usr.HomeDir, "/.kube/config"), + "kube config path, e.g. $HOME/.kube/config") + gsimage := flag.String("gameserver-image", "gcr.io/agones-images/cpp-simple-server:0.2", + "gameserver image to use for those tests, gcr.io/agones-images/cpp-simple-server:0.2") + + flag.Parse() + + var ( + err error + exitCode int + ) + + if framework, err = e2eframework.New(*kubeconfig, *gsimage); err != nil { + log.Printf("failed to setup framework: %v\n", err) + os.Exit(1) + } + defer func() { + err = framework.CleanUp(defaultNs) + if err != nil { + log.Printf("failed to cleanup resources: %v\n", err) + } + os.Exit(exitCode) + }() + exitCode = m.Run() + +}