From 6ef60887fe04b6ce6fcca6137c6bb74f909e08e9 Mon Sep 17 00:00:00 2001
From: Paul Wallrabe <54737071+raballew@users.noreply.github.com>
Date: Thu, 15 Oct 2020 07:13:26 +0000
Subject: [PATCH 1/4] Avoid SPOF for webhook

At the moment, the webhook is a SPOF in certain
scenarios. Under high load or when a node failure
occurs the webhook becomes unavailable. Defining
a HPA, PDB and affinity rules solves this issue.
---
 config/webhook-hpa.yaml | 65 +++++++++++++++++++++++++++++++++++++++++
 config/webhook.yaml     | 12 ++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 config/webhook-hpa.yaml

diff --git a/config/webhook-hpa.yaml b/config/webhook-hpa.yaml
new file mode 100644
index 00000000000..a0d8ddd86e8
--- /dev/null
+++ b/config/webhook-hpa.yaml
@@ -0,0 +1,65 @@
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: tekton-pipelines-webhook
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/name: webhook
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/version: "devel"
+    app.kubernetes.io/part-of: tekton-pipelines
+    # tekton.dev/release value replaced with inputs.params.versionTag in pipeline/tekton/publish.yaml
+    pipeline.tekton.dev/release: "devel"
+    # labels below are related to istio and should not be used for resource lookup
+    version: "devel"
+spec:
+  minReplicas: 1
+  maxReplicas: 5
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: tekton-pipelines-webhook
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        targetAverageUtilization: 100
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: tekton-pipelines-webhook
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/name: webhook
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/version: "devel"
+    app.kubernetes.io/part-of: tekton-pipelines
+    # tekton.dev/release value replaced with inputs.params.versionTag in pipeline/tekton/publish.yaml
+    pipeline.tekton.dev/release: "devel"
+    # labels below are related to istio and should not be used for resource lookup
+    version: "devel"
+spec:
+  minAvailable: 80%
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: webhook
+      app.kubernetes.io/component: webhook
+      app.kubernetes.io/instance: default
+      app.kubernetes.io/part-of: tekton-pipelines
diff --git a/config/webhook.yaml b/config/webhook.yaml
index 20642ea9c90..9047e32f6fd 100644
--- a/config/webhook.yaml
+++ b/config/webhook.yaml
@@ -54,6 +54,18 @@ spec:
         app: tekton-pipelines-webhook
         version: "devel"
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/name: webhook
+                  app.kubernetes.io/component: webhook
+                  app.kubernetes.io/instance: default
+                  app.kubernetes.io/part-of: tekton-pipelines
+              topologyKey: kubernetes.io/hostname
+            weight: 100
       serviceAccountName: tekton-pipelines-webhook
       containers:
       - name: webhook

From c0a4676a0a3faa849e73c5eba6db6e807f2c4750 Mon Sep 17 00:00:00 2001
From: Paul Wallrabe <54737071+raballew@users.noreply.github.com>
Date: Fri, 16 Oct 2020 10:55:39 +0000
Subject: [PATCH 2/4] Bump copyright year

Technically, the copyright year should have a value equal to the year of
the last contribution made to a file.
---
 config/webhook-hpa.yaml | 2 +-
 config/webhook.yaml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/webhook-hpa.yaml b/config/webhook-hpa.yaml
index a0d8ddd86e8..8c120723711 100644
--- a/config/webhook-hpa.yaml
+++ b/config/webhook-hpa.yaml
@@ -1,4 +1,4 @@
-# Copyright 2019 The Tekton Authors
+# Copyright 2020 The Tekton Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/config/webhook.yaml b/config/webhook.yaml
index 9047e32f6fd..d3ed9809d42 100644
--- a/config/webhook.yaml
+++ b/config/webhook.yaml
@@ -1,4 +1,4 @@
-# Copyright 2019 The Tekton Authors
+# Copyright 2020 The Tekton Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From 191214cafdafcb6eed1dc1d1c03f8eb97be3430c Mon Sep 17 00:00:00 2001
From: Paul Wallrabe <54737071+raballew@users.noreply.github.com>
Date: Fri, 16 Oct 2020 11:41:36 +0000
Subject: [PATCH 3/4] Add resource requests and limits for webhook

A resource request is required for autoscaler to take any action for a
metric.
---
 config/webhook.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/config/webhook.yaml b/config/webhook.yaml
index d3ed9809d42..97d9adad011 100644
--- a/config/webhook.yaml
+++ b/config/webhook.yaml
@@ -72,6 +72,14 @@ spec:
         # This is the Go import path for the binary that is containerized
         # and substituted here.
         image: ko://github.com/tektoncd/pipeline/cmd/webhook
+        # Resource request required for autoscaler to take any action for a metric
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 500Mi
         env:
         - name: SYSTEM_NAMESPACE
           valueFrom:

From 190cb55680a6bce51eea7c9de84c99f6625dd24c Mon Sep 17 00:00:00 2001
From: Paul Wallrabe <54737071+raballew@users.noreply.github.com>
Date: Mon, 19 Oct 2020 18:20:18 +0000
Subject: [PATCH 4/4] Add HA documentation

---
 docs/install.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/install.md b/docs/install.md
index fe7c1a8bb36..a15863f929a 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -19,6 +19,8 @@ This guide explains how to install Tekton Pipelines. It covers the following top
    will create a cluster running locally, with RBAC enabled and your user granted
    the `cluster-admin` role.
 
+1. If you want to support high availability usecases, install a [Metrics Server](https://github.com/kubernetes-sigs/metrics-server) on your cluster. 
+
 1. Choose the version of Tekton Pipelines you want to install. You have the following options:
 
    * **[Official](https://github.com/tektoncd/pipeline/releases)** - install this unless you have
@@ -349,6 +351,12 @@ data:
 
 You can create a custom release of Tekton Pipelines by following and customizing the steps in [Creating an official release](https://github.com/tektoncd/pipeline/blob/master/tekton/README.md#create-an-official-release). For example, you might want to customize the container images built and used by Tekton Pipelines.
 
+## Configuring High Availability
+
+If you want to run Tekton Pipelines in a way so that webhooks are resiliant against failures and support high concurrency scenarios, you need to run a [Metrics Server](https://github.com/kubernetes-sigs/metrics-server) in your Kubernetes cluster. This is required by the [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to compute the replica count. 
+
+**Note:** The default configuration is defined in [webhook-hpa.yaml](./../config/webhook-hpa.yaml) which could be customized to better fit a  specific usecase.
+
 ## Next steps
 
 To get started with Tekton Pipelines, see the [Tekton Pipelines Tutorial](./tutorial.md) and take a look at our [examples](https://github.com/tektoncd/pipeline/tree/master/examples).