From 80fd661eb63688965a57e7f658020c8e65bded24 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Tue, 8 Jan 2019 11:14:26 +0000 Subject: [PATCH] Helm port configuration improvements and unit tests (#3264) * Simplify port config and add unit tests * Add travis job for helm unit tests * Debug helm plugin issues * Fix node port * Follow best practice from the Prometheus chart * Address PR feedback * Fix typo * USe our own fork of helm-unittest --- .travis.yml | 3 + deploy/ci/travis/helm-chart-unit-tests.sh | 26 ++ deploy/kubernetes/README.md | 35 +-- deploy/kubernetes/console/.helmignore | 1 + .../console/templates/__helpers.tpl | 37 ++- .../kubernetes/console/templates/service.yaml | 59 ++-- .../console/tests/service_test.yaml | 284 ++++++++++++++++++ deploy/kubernetes/console/values.yaml | 18 +- 8 files changed, 409 insertions(+), 54 deletions(-) create mode 100755 deploy/ci/travis/helm-chart-unit-tests.sh create mode 100644 deploy/kubernetes/console/tests/service_test.yaml diff --git a/.travis.yml b/.travis.yml index b917289c96..42d8f832b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,6 +59,9 @@ jobs: - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh script: - npm run test-backend + - name: Helm Chart Unit Tests + script: + - "./deploy/ci/travis/helm-chart-unit-tests.sh" - name: E2E Tests - Long Suite before_script: - "./deploy/ci/travis/job-e2e-before_script.sh" diff --git a/deploy/ci/travis/helm-chart-unit-tests.sh b/deploy/ci/travis/helm-chart-unit-tests.sh new file mode 100755 index 0000000000..aa1d106565 --- /dev/null +++ b/deploy/ci/travis/helm-chart-unit-tests.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +echo "Stratos Helm Chart Unit Tests" +echo "=============================" + +echo "Installing Helm" +curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > get_helm.sh +chmod 700 get_helm.sh +./get_helm.sh + +echo "Helm Init (Client)" +helm init --client-only + +helm version --client + +echo "Install Helm unit test plugin" +helm plugin install https://github.com/cf-stratos/helm-unittest + +# Run unit tests +cd deploy/kubernetes +helm unittest console + +# Run lint +helm lint console diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md index 942bb936ac..be8bd2be94 100644 --- a/deploy/kubernetes/README.md +++ b/deploy/kubernetes/README.md @@ -172,10 +172,10 @@ To login use the following credentials detailed [here](../../docs/access.md). ## Advanced Topics ### Using a Load Balancer -If your Kubernetes deployment supports automatic configuration of a load balancer (e.g. Google Container Engine), specify the parameters `useLb=true` when installing. +If your Kubernetes deployment supports automatic configuration of a load balancer (e.g. Google Container Engine), specify the parameters `console.service.type=LoadBalancer` when installing. ``` -helm install stratos/console --namespace=console --name my-console --set useLb=true +helm install stratos/console --namespace=console --name my-console --set console.service.type=LoadBalancer ``` ### Specifying an External IP @@ -183,7 +183,7 @@ helm install stratos/console --namespace=console --name my-console --set useLb=t If the kubernetes cluster supports external IPs for services (see [ Service External IPs](https://kubernetes.io/docs/concepts/services-networking/service/#external-ips)), then the following arguments can be provided. In this following example the dashboard will be available at `https://192.168.100.100:5000`. ``` -helm install stratos/console --namespace=console --name my-console --set console.externalIP=192.168.100.100 console.port=5000 +helm install stratos/console --namespace=console --name my-console --set console.service.externalIPs={192.168.100.100} --set console.service.servicePort=5000 ``` ### Upgrading your deployment @@ -351,32 +351,3 @@ Install ``` helm install stratos/console --namespace=console --name my-console --version 2.0.0-dev-9a5611dc ``` - -### Enabled Metrics - -[Stratos Metrics](https://github.com/suse/stratos-metrics) can be deployed as part of the Stratos helm chart. The following override file will configure the Metrics component to fetch metrics from a PCF Dev instance. - -Save the following to a file called `values.yaml` -``` -consoleVersion: 2.0.0 -metrics: - enabled: true - env: - CLUSTER_ADMIN_PASSWORD: admin - UAA_CF_IDENTITY_ZONE: uaa - DOMAIN: local.pcfdev.io - UAA_ADMIN_CLIENT_SECRET: admin-client-secret - UAA_HOST: uaa.local.pcfdev.io - UAA_PORT: 443 - DOPPLER_PORT: 443 - firehoseExporter: - noIdentityZone: true -``` - -Deploy Stratos with Metrics enabled -``` -$ helm install ./console -f values.yaml --name console --namespace stratos - -``` - -The metrics endpoint will be available as `https://console-metrics-nginx`. This can registered as an endpoint in Stratos as a `Metrics` type. \ No newline at end of file diff --git a/deploy/kubernetes/console/.helmignore b/deploy/kubernetes/console/.helmignore index f0c1319444..79d2e7d312 100644 --- a/deploy/kubernetes/console/.helmignore +++ b/deploy/kubernetes/console/.helmignore @@ -19,3 +19,4 @@ .project .idea/ *.tmproj +tests/ \ No newline at end of file diff --git a/deploy/kubernetes/console/templates/__helpers.tpl b/deploy/kubernetes/console/templates/__helpers.tpl index 756fd53477..e7ef1cfec2 100644 --- a/deploy/kubernetes/console/templates/__helpers.tpl +++ b/deploy/kubernetes/console/templates/__helpers.tpl @@ -6,12 +6,14 @@ Image pull secret {{- printf "{\"%s\":{\"username\": \"%s\",\"password\":\"%s\",\"email\":\"%s\",\"auth\": \"%s\"}}" .Values.kube.registry.hostname .Values.kube.registry.username .Values.kube.registry.password .Values.kube.registry.email (printf "%s:%s" .Values.kube.registry.username .Values.kube.registry.password | b64enc) | b64enc }} {{- end }} + {{/* -Determine external IP: +Determine external IPs: This will do the following: 1. Check for Legacy SCF Config format 2. Check for Console specific External IP -3. Check for New SCf Config format +3. Check for New SCF Config format +4. Check for new Console External IPS */}} {{- define "service.externalIPs" -}} {{- if .Values.kube.external_ip -}} @@ -22,9 +24,14 @@ This will do the following: {{- range .Values.kube.external_ips -}} {{ printf "\n- %s" . | indent 4 -}} {{- end -}} +{{- else if .Values.console.service.externalIPs -}} +{{- range .Values.console.service.externalIPs -}} +{{ printf "\n- %s" . | indent 4 -}} +{{- end -}} {{- end -}} {{- end -}} + {{/* Get SCf UAA Endpoint */}} @@ -34,4 +41,28 @@ Get SCf UAA Endpoint {{- else if .Values.env.UAA_HOST -}} {{- printf "https://scf.%s:%v" .Values.env.UAA_HOST .Values.env.UAA_PORT -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + + +{{/* +Service type: +*/}} +{{- define "service.serviceType" -}} +{{- if or .Values.useLb .Values.services.loadbalanced -}} +LoadBalancer +{{- else -}} +{{- printf "%s" .Values.console.service.type -}} +{{- end -}} +{{- end -}} + + +{{/* +Service port: +*/}} +{{- define "service.servicePort" -}} +{{- if and .Values.kube.external_ips .Values.kube.external_console_https_port -}} +{{ printf "%v" .Values.kube.external_console_https_port }} +{{- else -}} +{{ .Values.console.service.servicePort }} +{{- end -}} +{{- end -}} diff --git a/deploy/kubernetes/console/templates/service.yaml b/deploy/kubernetes/console/templates/service.yaml index b3dcd7f3e2..c5a0227846 100644 --- a/deploy/kubernetes/console/templates/service.yaml +++ b/deploy/kubernetes/console/templates/service.yaml @@ -1,6 +1,10 @@ apiVersion: v1 kind: Service metadata: +{{- if .Values.console.service.annotations }} + annotations: +{{ toYaml .Values.console.service.annotations | indent 4 }} +{{- end }} labels: app.kubernetes.io/name: "stratos" app.kubernetes.io/instance: "{{ .Release.Name }}" @@ -9,32 +13,51 @@ metadata: helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" name: "{{ .Release.Name }}-ui-ext" spec: + selector: + app: "{{ .Release.Name }}" + component: console +{{- if .Values.console.service.clusterIP }} + clusterIP: {{ .Values.console.service.clusterIP }} +{{- end }} +{{- if or .Values.console.service.externalIPs .Values.kube.external_ip .Values.console.externalIP .Values.kube.external_ips }} + externalIPs: + {{ template "service.externalIPs" . }} +{{- end }} +{{- if .Values.console.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.console.service.loadBalancerIP }} +{{- end }} +{{- if .Values.console.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.console.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.console.service.externalName }} + externalName: {{ .Values.console.service.externalName }} +{{- end }} + +# Note: HTTP Port is optional - HTTPS port is always included +# HTTP Service (optional) ports: -{{- if .Values.kube.console_http_port }} +{{- if .Values.console.service.http.enabled }} - name: http - port: {{ .Values.kube.console_http_port }} + port: {{ .Values.console.service.http.servicePort }} protocol: TCP targetPort: 80 + {{- if .Values.console.service.http.nodePort }} + nodePort: {{ .Values.console.service.http.nodePort }} + {{- end }} {{- end }} +# HTTPS Service - name: https -{{- if and (or .Values.kube.external_ip .Values.console.externalIP .Values.kube.external_ips) .Values.kube.external_console_https_port }} - port: {{ .Values.kube.external_console_https_port }} -{{- else }} - port: {{ .Values.console.port }} -{{- end }} + port: {{ template "service.servicePort" . }} protocol: TCP targetPort: 443 - selector: - app: "{{ .Release.Name }}" - component: console -{{- if or .Values.useLb .Values.services.loadbalanced }} - type: LoadBalancer -{{- else }} - type: NodePort -{{- end }} -{{- if or .Values.kube.external_ip .Values.console.externalIP .Values.kube.external_ips }} - externalIPs: {{ template "service.externalIPs" . }} -{{- end }} + {{- if .Values.console.service.nodePort }} + nodePort: {{ .Values.console.service.nodePort }} + {{- end }} + type: {{ template "service.serviceType" . }} + --- apiVersion: v1 kind: Service diff --git a/deploy/kubernetes/console/tests/service_test.yaml b/deploy/kubernetes/console/tests/service_test.yaml new file mode 100644 index 0000000000..3fdb2867da --- /dev/null +++ b/deploy/kubernetes/console/tests/service_test.yaml @@ -0,0 +1,284 @@ +suite: test stratos external service +templates: + - service.yaml + +tests: + - it: should be a Service + asserts: + - isKind: + of: Service + - equal: + path: kind + value: Service + - it: should have default ClusterIP configuration + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.type + value: ClusterIP + - contains: + path: spec.ports + content: + name: https + port: 443 + protocol: TCP + targetPort: 443 + - it: should allow ClusterIP configuration with servicePort override + set: + console.service.servicePort: 12443 + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.type + value: ClusterIP + - contains: + path: spec.ports + content: + name: https + port: 12443 + protocol: TCP + targetPort: 443 + + - it: should allow nodePort configuration + set: + console.service.type: NodePort + asserts: + - equal: + path: spec.type + value: NodePort + - contains: + path: spec.ports + content: + name: https + port: 443 + protocol: TCP + targetPort: 443 + - it: should allow nodePort configuration port override + set: + console.service.type: NodePort + console.service.nodePort: 12345 + asserts: + - equal: + path: spec.type + value: NodePort + - contains: + path: spec.ports + content: + name: https + nodePort: 12345 + port: 443 + protocol: TCP + targetPort: 443 + - it: should allow nodePort configuration http + set: + console.service.type: NodePort + console.service.http.enabled: true + console.service.http.nodePort: 12345 + asserts: + - equal: + path: spec.type + value: NodePort + - contains: + path: spec.ports + content: + name: http + port: 80 + nodePort: 12345 + protocol: TCP + targetPort: 80 + - contains: + path: spec.ports + content: + name: https + port: 443 + protocol: TCP + targetPort: 443 + - it: should have Cluser IP set + set: + console.service.clusterIP: 5.6.7.8 + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.clusterIP + value: 5.6.7.8 + - it: should have LoadBalancer configuration + set: + useLb: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.type + value: LoadBalancer + - it: should have LoadBalancer configuration with IP + set: + console.service.type: LoadBalancer + console.service.loadBalancerIP: 1.2.3.4 + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: spec.loadBalancerIP + value: 1.2.3.4 + - it: should have LoadBalancer configuration (Legacy) + set: + services.loadbalanced: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.type + value: LoadBalancer + - it: should allow clusterIP configuration https + set: + console.https_port: + asserts: + - equal: + path: spec.type + value: ClusterIP + - contains: + path: spec.ports + content: + name: https + port: 443 + protocol: TCP + targetPort: 443 + - it: should allow clusterIP configuration http + set: + console.https_port: + console.service.http.enabled: true + asserts: + - equal: + path: spec.type + value: ClusterIP + - contains: + path: spec.ports + content: + name: http + port: 80 + protocol: TCP + targetPort: 80 + - contains: + path: spec.ports + content: + name: https + port: 443 + protocol: TCP + targetPort: 443 + - it: should allow external IP configuration (console.service.externalIPs) + set: + console.service.externalIPs: [ 1.2.3.4, 5.6.7.8 ] + asserts: + - contains: + path: spec.externalIPs + content: + 1.2.3.4 + - contains: + path: spec.externalIPs + content: + 5.6.7.8 + - it: should allow external IP configuration (console.externalIP) + set: + console.externalIP: 4.5.6.7 + asserts: + - contains: + path: spec.externalIPs + content: + 4.5.6.7 + - it: should allow external IP configuration (kube.external_ip) + set: + kube.external_ip: 4.5.6.7 + asserts: + - contains: + path: spec.externalIPs + content: + 4.5.6.7 + - it: should allow external IP configuration (kube.external_ips) + set: + kube.external_ips: [ 1.2.3.4, 5.6.7.8 ] + asserts: + - contains: + path: spec.externalIPs + content: + 1.2.3.4 + - contains: + path: spec.externalIPs + content: + 5.6.7.8 + - it: should allow external name configuration + set: + console.service.externalName: test_name + asserts: + - equal: + path: spec.externalName + value: test_name + - it: should allow annotations + set: + console.service.annotations: + - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + asserts: + - contains: + path: metadata.annotations + content: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + - it: should allow annotations (multiple) + set: + console.service.annotations: + - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + - service.beta.kubernetes.io/aws-load-balancer-connection-draining-timeout: "60" + asserts: + - contains: + path: metadata.annotations + content: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + - contains: + path: metadata.annotations + content: + service.beta.kubernetes.io/aws-load-balancer-connection-draining-timeout: "60" + - it: should support legacy service port configuration + set: + kube.external_ips: [ 1.2.3.4, 5.6.7.8 ] + kube.external_console_https_port: 8910 + console.service.servicePort: 22233 + asserts: + - contains: + path: spec.externalIPs + content: + 1.2.3.4 + - contains: + path: spec.ports + content: + name: https + port: 8910 + protocol: TCP + targetPort: 443 + - it: should support legacy service port configuration (only with kube.external_ips) + set: + kube.external_console_https_port: 8910 + console.service.servicePort: 22233 + asserts: + - contains: + path: spec.ports + content: + name: https + port: 22233 + protocol: TCP + targetPort: 443 + - it: should support legacy default service port configuration (only with kube.external_ips) + set: + kube.external_ips: [ 1.2.3.4, 5.6.7.8 ] + console.service.servicePort: 22233 + asserts: + - contains: + path: spec.ports + content: + name: https + port: 8443 + protocol: TCP + targetPort: 443 diff --git a/deploy/kubernetes/console/values.yaml b/deploy/kubernetes/console/values.yaml index b0b486b050..0740cb11e3 100644 --- a/deploy/kubernetes/console/values.yaml +++ b/deploy/kubernetes/console/values.yaml @@ -11,14 +11,30 @@ dbProvider: mysql #ftpProxy: proxy.corp.net #socksProxy: sock-proxy.corp.net imagePullPolicy: IfNotPresent +# useLb is deprecated - use console.service.type useLb: false console: - port: 443 cookieDomain: + # externalIP deprecated - use console.service.externalIPs # externalIP: 127.0.0.1 backendLogLevel: info ssoLogin: false ssoOptions: + + # Stratos Services + service: + annotations: [] + externalIPs: [] + loadBalancerIP: + loadBalancerSourceRanges: [] + servicePort: 443 + # nodePort: 30000 + type: ClusterIP + externalName: + http: + enabled: false + servicePort: 80 + # nodePort: 30001 images: console: stratos-console proxy: stratos-jetstream