Add CircleCI workflow for end-to-end testing with Kubernetes Kind

.circleci/config.yml

@@ -0,0 +1,16 @@
+version: 2.1
+jobs:
+  e2e-testing:
+    machine: true
+    steps:
+      - checkout
+      - run: test/e2e-kind.sh
+      - run: test/e2e-istio.sh
+      - run: test/e2e-build.sh
+      - run: test/e2e-tests.sh
+
+workflows:
+  version: 2
+  build-and-test:
+    jobs:
+      - e2e-testing

.travis.yml

@@ -12,12 +12,17 @@ addons:
     packages:
     - docker-ce
 
+#before_script:
+#  - go get -u sigs.k8s.io/kind
+#  - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+#  - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+
 script:
-- set -e
-- make test-fmt
-- make test-codegen
-- go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/
-- make build
+  - set -e
+  - make test-fmt
+  - make test-codegen
+  - go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/
+  - make build
 
 after_success:
 - if [ -z "$DOCKER_USER" ]; then

artifacts/flagger/deployment.yaml

@@ -23,7 +23,7 @@ spec:
       containers:
       - name: flagger
         image: quay.io/stefanprodan/flagger:0.5.1
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         ports:
         - name: http
           containerPort: 8080

docs/gitbook/usage/progressive-delivery.md

@@ -113,7 +113,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.0
+podinfod=quay.io/stefanprodan/podinfo:1.4.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

test/Dockerfile.kind

@@ -0,0 +1,4 @@
+FROM golang:1.11
+
+RUN go get -u sigs.k8s.io/kind
+

test/README.md

@@ -0,0 +1,24 @@
+# Flagger end-to-end testing
+
+The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind).
+
+CircleCI e2e workflow:
+
+* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh)
+* build Kubernetes Kind from master [e2e-kind.sh](e2e-kind.sh)
+* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh)
+* install latest stable Helm CLI [e2e-istio.sh](e2e-istio.sh)
+* deploy Tiller on the local cluster [e2e-istio.sh](e2e-istio.sh)
+* install Istio CRDs with Helm [e2e-istio.sh](e2e-istio.sh)
+* install Istio control plane and Prometheus with Helm [e2e-istio.sh](e2e-istio.sh)
+* build Flagger container image [e2e-build.sh](e2e-build.sh)
+* load Flagger image onto the local cluster [e2e-build.sh](e2e-build.sh)
+* deploy Flagger in the istio-system namespace [e2e-build.sh](e2e-build.sh)
+* create a test namespace with Istio injection enabled [e2e-tests.sh](e2e-tests.sh)
+* deploy the load tester in the test namespace [e2e-tests.sh](e2e-tests.sh)
+* deploy a demo workload (podinfo) in the test namespace [e2e-tests.sh](e2e-tests.sh)
+* test the canary initialization [e2e-tests.sh](e2e-tests.sh)
+* test the canary analysis and promotion [e2e-tests.sh](e2e-tests.sh)
+
+
+

test/e2e-build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Building Flagger'
+cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile
+
+echo '>>> Installing Flagger'
+kind load docker-image test/flagger:latest
+kubectl apply -f ${REPO_ROOT}/artifacts/flagger/
+kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest
+kubectl -n istio-system rollout status deployment/flagger

test/e2e-istio-values.yaml

@@ -0,0 +1,62 @@
+#
+# Minimal Istio Configuration required by Flagger
+#
+
+# pilot configuration
+pilot:
+  enabled: true
+  sidecar: true
+
+gateways:
+  enabled: false
+  istio-ingressgateway:
+    autoscaleMax: 1
+
+# citadel configuration
+security:
+  enabled: true
+
+# sidecar-injector webhook configuration
+sidecarInjectorWebhook:
+  enabled: true
+
+# galley configuration
+galley:
+  enabled: false
+
+# mixer configuration
+mixer:
+  policy:
+    enabled: false
+    replicaCount: 1
+    autoscaleEnabled: false
+  telemetry:
+    enabled: true
+    replicaCount: 1
+    autoscaleEnabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+
+# addon prometheus configuration
+prometheus:
+  enabled: true
+  scrapeInterval: 5s
+
+# addon jaeger tracing configuration
+tracing:
+  enabled: false
+
+# Common settings.
+global:
+  proxy:
+    # Resources for the sidecar.
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 2000m
+        memory: 128Mi
+  useMCP: false

test/e2e-istio.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+ISTIO_VER="1.1.0-rc.0"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo ">>> Installing Helm"
+curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+
+echo '>>> Installing Tiller'
+kubectl --namespace kube-system create sa tiller
+kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+helm init --service-account tiller --upgrade --wait
+
+echo ">>> Installing Istio ${ISTIO_VER}"
+helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts
+
+echo '>>> Installing Istio CRDs'
+helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system
+
+echo '>>> Waiting for Istio CRDs to be ready'
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11
+
+echo '>>> Installing Istio control plane'
+helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml

test/e2e-kind.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo ">>> Installing kubectl"
+curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
+chmod +x kubectl && \
+sudo mv kubectl /usr/local/bin/
+
+echo ">>> Building sigs.k8s.io/kind"
+docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile.kind
+docker create -ti --name dummy kind:src sh
+docker cp dummy:/go/bin/kind ./kind
+docker rm -f dummy
+
+echo ">>> Installing kind"
+chmod +x kind
+sudo mv kind /usr/local/bin/
+kind create cluster --wait 5m
+
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+kubectl get pods --all-namespaces
+

test/e2e-tests.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind, Helm and Istio
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+kubectl label namespace test istio-injection=enabled
+
+echo '>>> Installing the load tester'
+kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Initialising canary'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+  canaryAnalysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: istio_requests_total
+      threshold: 99
+      interval: 1m
+    - name: istio_request_duration_seconds_bucket
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test:9898/"
+
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+kubectl -n istio-system logs deployment/flagger
+
+echo '✔ All tests passed'

test/e2e-workload.yaml

@@ -0,0 +1,67 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: podinfo
+  namespace: test
+  labels:
+    app: podinfo
+spec:
+  minReadySeconds: 5
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: podinfo
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - name: podinfod
+        image: quay.io/stefanprodan/podinfo:1.4.0
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 9898
+          name: http
+          protocol: TCP
+        command:
+        - ./podinfo
+        - --port=9898
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          value: blue
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 1000m
+            memory: 128Mi
+          requests:
+            cpu: 1m
+            memory: 16Mi