feat: multi-tenant scaler and interceptor in the HTTP add-on

http-add-on/templates/crd.yaml

@@ -1,10 +1,9 @@
-{{- if .Values.crds.install }}
-
+{{ if .Values.crds.install }}
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.5.0
+    controller-gen.kubebuilder.io/version: v0.6.1
   creationTimestamp: null
   name: httpscaledobjects.http.keda.sh
 spec:
@@ -59,6 +58,11 @@ spec:
           spec:
             description: HTTPScaledObjectSpec defines the desired state of HTTPScaledObject
             properties:
+              host:
+                description: The host to route. All requests with this host in the
+                  "Host" header will be routed to the Service and Port specified in
+                  the scaleTargetRef
+                type: string
               replicas:
                 description: (optional) Replica information
                 properties:
@@ -93,11 +97,8 @@ spec:
                 - port
                 - service
                 type: object
-              targetMetric:
-                description: (optional) Target metric value
-                format: int32
-                type: integer
             required:
+            - host
             - scaleTargetRef
             type: object
           status:
@@ -115,32 +116,11 @@ spec:
                     reason:
                       description: The reason for the condition's last transition.
                       enum:
-                      - ErrorCreatingExternalScaler
-                      - ErrorCreatingExternalScalerService
-                      - CreatedExternalScaler
-                      - ErrorCreatingInterceptorScaledObject
                       - ErrorCreatingAppScaledObject
                       - AppScaledObjectCreated
-                      - InterceptorScaledObjectCreated
-                      - ErrorCreatingInterceptor
-                      - ErrorCreatingInterceptorAdminService
-                      - ErrorCreatingInterceptorProxyService
-                      - InterceptorCreated
                       - TerminatingResources
-                      - InterceptorDeploymentTerminated
-                      - InterceptorDeploymentTerminationError
-                      - InterceptorAdminServiceTerminationError
-                      - InterceptorAdminServiceTerminated
-                      - InterceptorProxyServiceTerminationError
-                      - InterceptorProxyServiceTerminated
-                      - ExternalScalerDeploymentTerminationError
-                      - ExternalScalerDeploymentTerminated
-                      - ExternalScalerServiceTerminationError
-                      - ExternalScalerServiceTerminated
-                      - InterceptorScaledObjectTerminated
                       - AppScaledObjectTerminated
                       - AppScaledObjectTerminationError
-                      - InterceptorScaledObjectTerminationError
                       - PendingCreation
                       - HTTPScaledObjectIsReady
                       type: string
@@ -178,5 +158,4 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
-
-{{- end -}}
+{{ end }}

http-add-on/templates/deployment-interceptor.yaml

@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: interceptor
+    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    keda.sh/addon: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
+    name: {{ .Chart.Name }}-interceptor
+    app.kubernetes.io/name: {{ .Chart.Name }}-interceptor
+    app.kubernetes.io/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    app.kubernetes.io/component: interceptor
+    app.kubernetes.io/part-of: {{ .Chart.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{ tpl .Values.additionalLabels . | indent 4}}
+  name: {{ .Chart.Name }}-interceptor
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: interceptor
+      httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+      keda.sh/addon: {{ .Chart.Name }}
+  template:
+    metadata:
+      labels:
+        control-plane: interceptor
+        httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+        keda.sh/addon: {{ .Chart.Name }}
+        app: {{ .Chart.Name }}
+        name: {{ .Chart.Name }}-interceptor
+        app.kubernetes.io/name: {{ .Chart.Name }}-interceptor
+        app.kubernetes.io/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+        app.kubernetes.io/component: interceptor
+        app.kubernetes.io/part-of: {{ .Chart.Name }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    {{ tpl .Values.additionalLabels . | indent 4}}
+    spec:
+      containers:
+      - args:
+        image: "{{ .Values.images.interceptor }}:{{ .Values.images.tag | default .Chart.AppVersion }}"
+        imagePullPolicy: '{{ .Values.interceptor.pullPolicy | default "Always" }}'
+        name: "{{ .Chart.Name }}-interceptor"
+        env:
+        # - name: KEDA_HTTP_OPERATOR_SERVICE_NAME
+        #   value: "{{ .Chart.Name }}-{{ .Values.operator.adminService }}"
+        # - name: KEDA_HTTP_OPERATOR_SERVICE_PORT
+        #   value: "{{ .Values.operator.adminPort }}"
+        # - name: KEDA_HTTP_OPERATOR_ROUTING_TABLE_PATH
+        #   value: "{{ .Values.operator.routingTableURLPath }}"
+        - name: KEDA_HTTP_ROUTING_TABLE_UPDATE_DURATION_MS
+          value: "{{ .Values.interceptor.routingTableUpdateDurationMS }}"
+        - name: KEDA_HTTP_CURRENT_NAMESPACE
+          value: "{{ .Release.Namespace }}"
+        - name: KEDA_HTTP_PROXY_PORT
+          value: "{{ .Values.interceptor.proxyPort }}"
+        - name: KEDA_HTTP_ADMIN_PORT
+          value: "{{ .Values.interceptor.adminPort }}"
+        - name: KEDA_HTTP_CONNECT_TIMEOUT
+          value: "{{ .Values.interceptor.tcpConnectTimeout }}"
+        - name: KEDA_HTTP_KEEP_ALIVE
+          value: "{{ .Values.interceptor.keepAlive }}"
+        - name: KEDA_RESPONSE_HEADER_TIMEOUT
+          value: "{{ .Values.interceptor.responseHeaderTimeout }}"
+        - name: KEDA_CONDITION_WAIT_TIMEOUT
+          value: "{{ .Values.interceptor.replicasWaitTimeout }}"
+        - name: KEDA_HTTP_DEPLOYMENT_CACHE_POLLING_INTERVAL_MS
+          value: "{{ .Values.interceptor.deploymentCachePollingIntervalMS }}"
+        - name: KEDA_HTTP_FORCE_HTTP2
+          value: "{{ .Values.interceptor.forceHTTP2 }}"
+        - name: KEDA_HTTP_MAX_IDLE_CONNS
+          value: "{{ .Values.interceptor.maxIdleConns }}"
+        - name: KEDA_HTTP_IDLE_CONN_TIMEOUT
+          value: "{{ .Values.interceptor.idleConnTimeout }}"
+        - name: KEDA_HTTP_TLS_HANDSHAKE_TIMEOUT
+          value: "{{ .Values.interceptor.tlsHandshakeTimeout }}"
+        - name: KEDA_HTTP_EXPECT_CONTINUE_TIMEOUT
+          value: "{{ .Values.interceptor.expectContinueTimeout }}"
+        ports:
+        - containerPort: {{ .Values.interceptor.adminPort }}
+          name: inter-admin
+        - containerPort: {{ .Values.interceptor.proxyPort }}
+          name: inter-proxy
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+      terminationGracePeriodSeconds: 10

http-add-on/templates/deployment.yaml → http-add-on/templates/deployment-operator.yaml

@@ -50,18 +50,25 @@ spec:
       - args:
         - --metrics-addr=127.0.0.1:8080
         - --enable-leader-election
+        - --admin-port={{ .Values.operator.adminPort }}
         image: "{{ .Values.images.operator }}:{{ .Values.images.tag | default .Chart.AppVersion }}"
-        imagePullPolicy: {{ .Values.operator.pullPolicy | default "Always" }}
-        name: {{ .Chart.Name }}-operator
+        imagePullPolicy: '{{ .Values.operator.pullPolicy | default "Always" }}'
+        name: "{{ .Chart.Name }}-operator"
         env:
-        - name: KEDAHTTP_OPERATOR_EXTERNAL_SCALER_IMAGE
-          value: "{{ .Values.images.scaler }}:{{ .Values.images.tag | default .Chart.AppVersion }}"
-        - name: KEDAHTTP_OPERATOR_INTERCEPTOR_IMAGE
-          value: "{{ .Values.images.interceptor }}:{{ .Values.images.tag | default .Chart.AppVersion }}"
-        - name: INTERCEPTOR_PULL_POLICY
-          value: {{ .Values.operator.pullPolicy | default "Always" }}
-        - name: SCALER_PULL_POLICY
-          value: {{ .Values.operator.pullPolicy | default "Always"}}
+        - name: KEDAHTTP_INTERCEPTOR_SERVICE
+          value: "{{.Chart.Name}}-{{ .Values.interceptor.adminService }}"
+        - name: KEDAHTTP_OPERATOR_EXTERNAL_SCALER_SERVICE
+          value: "{{ .Chart.Name }}-{{ .Values.scaler.service }}"
+        - name: KEDAHTTP_OPERATOR_EXTERNAL_SCALER_PORT
+          value: "{{ default 9090 .Values.scaler.grpcPort }}"
+        - name: KEDAHTTP_INTERCEPTOR_ADMIN_PORT
+          value: "{{ .Values.interceptor.adminPort }}"
+        - name: KEDAHTTP_INTERCEPTOR_PROXY_PORT
+          value: "{{ .Values.interceptor.proxyPort }}"
+
+        ports: 
+        - containerPort: {{ .Values.operator.adminPort }}
+          name: admin-http
         resources:
           {{- toYaml .Values.resources | nindent 12 }}
       terminationGracePeriodSeconds: 10

http-add-on/templates/deployment-scaler.yaml

@@ -0,0 +1,67 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: external-scaler
+    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    keda.sh/addon: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
+    name: {{ .Chart.Name }}-external-scaler
+    app.kubernetes.io/name: {{ .Chart.Name }}-external-scaler
+    app.kubernetes.io/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    app.kubernetes.io/component: external-scaler
+    app.kubernetes.io/part-of: {{ .Chart.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{ tpl .Values.additionalLabels . | indent 4}}
+  name: {{ .Chart.Name }}-external-scaler
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: external-scaler
+      httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+      keda.sh/addon: {{ .Chart.Name }}
+  template:
+    metadata:
+      labels:
+        control-plane: external-scaler
+        httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+        keda.sh/addon: {{ .Chart.Name }}
+        app: {{ .Chart.Name }}
+        name: "{{ .Chart.Name }}-external-scaler"
+        app.kubernetes.io/name: "{{ .Chart.Name }}-external-scaler"
+        app.kubernetes.io/version: "{{ .Values.images.tag | default .Chart.AppVersion }}"
+        app.kubernetes.io/component: external-scaler
+        app.kubernetes.io/part-of: {{ .Chart.Name }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    {{ tpl .Values.additionalLabels . | indent 4}}
+    spec:
+      containers:
+      - args:
+        image: "{{ .Values.images.scaler }}:{{ .Values.images.tag | default .Chart.AppVersion }}"
+        imagePullPolicy: {{ .Values.scaler.pullPolicy | default "Always" }}
+        name: "{{ .Chart.Name }}-external-scaler"
+        ports:
+        - containerPort: {{ .Values.scaler.healthPort }}
+          name: scaler-health
+        - containerPort: {{ .Values.scaler.grpcPort }}
+          name: scaler-grpc
+        env:
+        - name: KEDA_HTTP_SCALER_PORT
+          value: "{{ .Values.scaler.grpcPort }}"
+        - name: KEDA_HTTP_HEALTH_PORT
+          value: "{{ .Values.scaler.healthPort }}"
+        - name: KEDA_HTTP_SCALER_TARGET_ADMIN_NAMESPACE
+          value: "{{ .Release.Namespace }}"
+        - name: KEDA_HTTP_SCALER_TARGET_ADMIN_SERVICE
+          value: "{{ .Chart.Name }}-{{ .Values.interceptor.adminService }}"
+        - name: KEDA_HTTP_SCALER_TARGET_ADMIN_PORT
+          value: "{{ default 9091 .Values.interceptor.adminPort }}"
+        - name: KEDA_HTTP_SCALER_TARGET_PENDING_REQUESTS_INTERCEPTOR
+          value: {{ default 200 .Values.scaler.pendingRequestsInterceptor }}
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+      terminationGracePeriodSeconds: 10

http-add-on/templates/routing-table-config-map.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: keda-http-routing-table
+  labels:
+    "control-plane": "operator"
+    "keda.sh/addon": "http-add-on"
+    "app":           "http-add-on"
+    "name":          "http-add-on-routing-table"
+data:

http-add-on/templates/scaledobject-interceptor.yaml

@@ -0,0 +1,17 @@
+# this is the ScaledObject that tells KEDA to scale the interceptor fleet
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: "{{ .Chart.Name }}-interceptor"
+spec:
+  minReplicaCount: {{ .Values.interceptor.minReplicas }}
+  maxReplicaCount: {{ .Values.interceptor.maxReplicas }}
+  pollingInterval: {{ .Values.interceptor.pollingInterval }}
+  scaleTargetRef:
+    name: "{{ .Chart.Name }}-interceptor"
+    kind: Deployment
+  triggers:
+    - type: external
+      metadata:
+        scalerAddress: "{{ .Chart.Name }}-{{ .Values.scaler.service }}:{{ default 9091 .Values.scaler.grpcPort }}"
+        host: interceptor

http-add-on/templates/svc-interceptor-admin.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: interceptor-admin
+    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    keda.sh/addon: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
+    name: "{{ .Chart.Name }}-{{ .Values.interceptor.adminService }}"
+    app.kubernetes.io/name: {{ .Chart.Name }}-interceptor-admin
+    app.kubernetes.io/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    app.kubernetes.io/component: interceptor-admin
+    app.kubernetes.io/part-of: {{ .Chart.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{ tpl .Values.additionalLabels . | indent 4}}
+  name: "{{ .Chart.Name }}-{{ .Values.interceptor.adminService }}"
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+  - name: https
+    port: {{ default 9091 .Values.interceptor.adminPort }}
+    targetPort: inter-admin
+  selector:
+    control-plane: interceptor

http-add-on/templates/svc-interceptor-proxy.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: interceptor-proxy
+    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    keda.sh/addon: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
+    name: "{{ .Chart.Name }}-{{ .Values.interceptor.proxyService }}"
+    app.kubernetes.io/name: {{ .Chart.Name }}-interceptor-proxy
+    app.kubernetes.io/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    app.kubernetes.io/component: interceptor-proxy
+    app.kubernetes.io/part-of: {{ .Chart.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{ tpl .Values.additionalLabels . | indent 4}}
+  name: "{{ .Chart.Name }}-{{ .Values.interceptor.proxyService }}"
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+  - name: https
+    port: {{ default 9091 .Values.interceptor.proxyPort }}
+    targetPort: inter-proxy
+  selector:
+    control-plane: interceptor

http-add-on/templates/svc-operator-admin.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: operator-admin
+    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
+    keda.sh/addon: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
+    name: "{{ .Chart.Name }}-{{ .Values.operator.adminService }}"
+    app.kubernetes.io/name: "{{ .Chart.Name }}-operator-admin"
+    app.kubernetes.io/version: "{{ .Values.images.tag | default .Chart.AppVersion }}"
+    app.kubernetes.io/component: operator-admin
+    app.kubernetes.io/part-of: {{ .Chart.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{ tpl .Values.additionalLabels . | indent 4}}
+  name: "{{ .Chart.Name }}-{{ .Values.operator.adminService }}"
+  namespace: "{{ .Release.Namespace }}"
+spec:
+  ports:
+  - name: admin-http
+    port: {{ default 9091 .Values.operator.adminPort }}
+    targetPort: admin-http
+  selector:
+    control-plane: controller-manager

http-add-on/templates/svc.yaml → http-add-on/templates/svc-operator-metrics.yaml

@@ -23,5 +23,3 @@ spec:
     targetPort: https
   selector:
     control-plane: controller-manager
-    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
-    keda.sh/addon: {{ .Chart.Name }}