feat: Use self-signed certs for TLS (between components and between k8s and KEDA)

CHANGELOG.md

@@ -58,7 +58,8 @@ Here is an overview of all new **experimental** features:
 
 ### Improvements
 
-- **Redis**: Add support to Redis 7 ([#4052](https://github.com/kedacore/keda/issues/4052))
+- **General**: Use (self-signed) certificates for all the communications (internals and externals) ([#3931](https://github.com/kedacore/keda/issues/3931))
+- **Redis Scalers**: Add support to Redis 7 ([#4052](https://github.com/kedacore/keda/issues/4052))
 
 ### Fixes
 

Dockerfile.adapter

@@ -20,8 +20,6 @@ COPY vendor/ vendor/
 COPY go.mod go.mod
 COPY go.sum go.sum
 
-RUN mkdir -p /apiserver.local.config/certificates && chmod -R 777 /apiserver.local.config
-
 # Build
 # https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
 ARG TARGETOS
@@ -32,7 +30,6 @@ RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION}
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder --chown=nonroot:nonroot /apiserver.local.config  /apiserver.local.config
 COPY --from=builder /workspace/bin/keda-adapter .
 # 65532 is numeric for nonroot
 USER 65532:65532

cmd/adapter/main.go

@@ -81,7 +81,6 @@ func (a *Adapter) makeProvider(ctx context.Context, globalHTTPTimeout time.Durat
 		logger.Error(err, "failed to add keda scheme to runtime scheme")
 		return nil, nil, fmt.Errorf("failed to add keda scheme to runtime scheme (%s)", err)
 	}
-
 	namespace, err := getWatchNamespace()
 	if err != nil {
 		logger.Error(err, "failed to get watch namespace")
@@ -160,7 +159,7 @@ func (a *Adapter) makeProvider(ctx context.Context, globalHTTPTimeout time.Durat
 	go func() { prometheusServer.NewServer(fmt.Sprintf(":%v", prometheusMetricsPort), prometheusMetricsPath) }()
 
 	logger.Info("Connecting Metrics Service gRPC client to the server", "address", metricsServiceAddr)
-	grpcClient, err := metricsservice.NewGrpcClient(metricsServiceAddr)
+	grpcClient, err := metricsservice.NewGrpcClient(metricsServiceAddr, a.SecureServing.ServerCert.CertDirectory)
 	if err != nil {
 		logger.Error(err, "error connecting Metrics Service gRPC client to the server", "address", metricsServiceAddr)
 		return nil, nil, err
@@ -244,6 +243,7 @@ func main() {
 	cmd.Flags().Float32Var(&adapterClientRequestQPS, "kube-api-qps", 20.0, "Set the QPS rate for throttling requests sent to the apiserver")
 	cmd.Flags().IntVar(&adapterClientRequestBurst, "kube-api-burst", 30, "Set the burst for throttling requests sent to the apiserver")
 	cmd.Flags().BoolVar(&disableCompression, "disable-compression", true, "Disable response compression for k8s restAPI in client-go. ")
+
 	if err := cmd.Flags().Parse(os.Args); err != nil {
 		return
 	}

cmd/operator/main.go

@@ -17,34 +17,28 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"os"
 	"runtime"
 	"time"
 
-	"github.com/open-policy-agent/cert-controller/pkg/rotator"
 	"github.com/spf13/pflag"
-	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/tools/cache"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	kedacontrollers "github.com/kedacore/keda/v2/controllers/keda"
+	"github.com/kedacore/keda/v2/pkg/certificates"
 	"github.com/kedacore/keda/v2/pkg/k8s"
 	"github.com/kedacore/keda/v2/pkg/metricsservice"
 	"github.com/kedacore/keda/v2/pkg/scaling"
@@ -88,8 +82,8 @@ func main() {
 	var operatorServiceName string
 	var metricsServerServiceName string
 	var webhooksServiceName string
-
 	var enableCertRotation bool
+	var validatingWebhookName string
 	pflag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	pflag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	pflag.StringVar(&metricsServiceAddr, "metrics-service-bind-address", ":9666", "The address the gRPRC Metrics Service endpoint binds to.")
@@ -104,9 +98,8 @@ func main() {
 	pflag.StringVar(&operatorServiceName, "operator-service-name", "keda-operator", "Operator service name. Defaults to keda-operator")
 	pflag.StringVar(&metricsServerServiceName, "metrics-server-service-name", "keda-metrics-apiserver", "Metrics server service name. Defaults to keda-metrics-apiserver")
 	pflag.StringVar(&webhooksServiceName, "webhooks-service-name", "keda-admission-webhooks", "Webhook service name. Defaults to keda-admission-webhooks")
-
 	pflag.BoolVar(&enableCertRotation, "enable-cert-rotation", false, "enable automatic generation and rotation of TLS certificates/keys")
-
+	pflag.StringVar(&validatingWebhookName, "validating-webhook-name", "keda-admission", "ValidatingWebhookConfiguration name. Defaults to keda-admission")
 	opts := zap.Options{}
 	opts.BindFlags(flag.CommandLine)
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
@@ -251,14 +244,30 @@ func main() {
 		os.Exit(1)
 	}
 
+	certReady := make(chan struct{})
 	if enableCertRotation {
-		if err := addCertificateRotation(ctx, mgr, certSecretName, certDir, operatorServiceName, metricsServerServiceName, webhooksServiceName); err != nil {
+		certManager := certificates.CertManager{
+			SecretName:            certSecretName,
+			CertDir:               certDir,
+			OperatorService:       operatorServiceName,
+			MetricsServerService:  metricsServerServiceName,
+			WebhookService:        webhooksServiceName,
+			CAName:                "KEDA",
+			CAOrganization:        "KEDAORG",
+			ValidatingWebhookName: validatingWebhookName,
+			APIServiceName:        "v1beta1.external.metrics.k8s.io",
+			Logger:                setupLog,
+			Ready:                 certReady,
+		}
+		if err := certManager.AddCertificateRotation(ctx, mgr); err != nil {
 			setupLog.Error(err, "unable to set up cert rotation")
 			os.Exit(1)
 		}
+	} else {
+		close(certReady)
 	}
 
-	grpcServer := metricsservice.NewGrpcServer(&scaledHandler, metricsServiceAddr)
+	grpcServer := metricsservice.NewGrpcServer(&scaledHandler, metricsServiceAddr, certDir, certReady)
 	if err := mgr.Add(&grpcServer); err != nil {
 		setupLog.Error(err, "unable to set up Metrics Service gRPC server")
 		os.Exit(1)
@@ -283,101 +292,3 @@ func main() {
 		os.Exit(1)
 	}
 }
-
-// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;update;patch
-// +kubebuilder:rbac:groups="",namespace=keda,resources=secrets,verbs=get;list;watch;create;update;patch;delete
-
-// addCertificateRotation registers all needed services to generate the certificates and patches needed resources with the caBundle
-func addCertificateRotation(ctx context.Context, mgr manager.Manager, secretName, certDir, operatorServiceName, metricsServerServiceName, webhooksServiceName string) error {
-	caName := "kedaorg-ca"
-	caOrganization := "kedaorg"
-
-	var rotatorHooks = []rotator.WebhookInfo{
-		{
-			Name: "keda-admission",
-			Type: rotator.Validating,
-		},
-	}
-
-	// Make sure certs are generated and valid if cert rotation is enabled.
-	setupFinished := make(chan struct{})
-	ensureSecret(ctx, mgr, secretName)
-	extraDNSNames := []string{}
-	extraDNSNames = append(extraDNSNames, getDNSNames(operatorServiceName)...)
-	extraDNSNames = append(extraDNSNames, getDNSNames(webhooksServiceName)...)
-	extraDNSNames = append(extraDNSNames, getDNSNames(metricsServerServiceName)...)
-
-	setupLog.V(1).Info("setting up cert rotation")
-	if err := rotator.AddRotator(mgr, &rotator.CertRotator{
-		SecretKey: types.NamespacedName{
-			Namespace: kedautil.GetPodNamespace(),
-			Name:      secretName,
-		},
-		CertDir:                certDir,
-		CAName:                 caName,
-		CAOrganization:         caOrganization,
-		DNSName:                extraDNSNames[0],
-		ExtraDNSNames:          extraDNSNames,
-		IsReady:                setupFinished,
-		Webhooks:               rotatorHooks,
-		RestartOnSecretRefresh: true,
-		RequireLeaderElection:  true,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-// getDNSNames  creates all the possible DNS names for a given service
-func getDNSNames(service string) []string {
-	namespace := kedautil.GetPodNamespace()
-	return []string{
-		service,
-		fmt.Sprintf("%s.%s", service, namespace),
-		fmt.Sprintf("%s.%s.svc", service, namespace),
-		fmt.Sprintf("%s.%s.svc.local", service, namespace),
-	}
-}
-
-// ensureSecret ensures that the secret used for storing TLS certificates exists
-func ensureSecret(ctx context.Context, mgr manager.Manager, secretName string) {
-	secrets := &corev1.SecretList{}
-	kedaNamespace := kedautil.GetPodNamespace()
-	opt := &client.ListOptions{
-		Namespace: kedaNamespace,
-	}
-
-	err := mgr.GetAPIReader().List(ctx, secrets, opt)
-	if err != nil {
-		setupLog.Error(err, "unable to check secrets")
-		os.Exit(1)
-	}
-
-	exists := false
-	for _, secret := range secrets.Items {
-		if secret.Name == secretName {
-			exists = true
-			break
-		}
-	}
-	if !exists {
-		secret := &corev1.Secret{
-			ObjectMeta: v1.ObjectMeta{
-				Name:      secretName,
-				Namespace: kedaNamespace,
-				Labels: map[string]string{
-					"app":                         "keda-operator",
-					"app.kubernetes.io/name":      "keda-operator",
-					"app.kubernetes.io/component": "keda-operator",
-					"app.kubernetes.io/part-of":   "keda",
-				},
-			},
-		}
-		err = mgr.GetClient().Create(ctx, secret)
-		if err != nil {
-			setupLog.Error(err, "unable to create certificates secret")
-			os.Exit(1)
-		}
-		setupLog.V(1).Info(fmt.Sprintf("created the secret %s to store cert-controller certificates", secretName))
-	}
-}

config/metrics-server/api_service.yaml

@@ -13,6 +13,5 @@ spec:
     namespace: keda
   group: external.metrics.k8s.io
   version: v1beta1
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 100
   versionPriority: 100

config/metrics-server/deployment.yaml

@@ -59,6 +59,10 @@ spec:
           - --secure-port=6443
           - --logtostderr=true
           - --v=0
+          - --client-ca-file=/certs/ca.crt
+          - --tls-cert-file=/certs/tls.crt
+          - --tls-private-key-file=/certs/tls.key
+          - --cert-dir=/certs
           ports:
           - containerPort: 6443
             name: https
@@ -69,8 +73,6 @@ spec:
           volumeMounts:
           - mountPath: /tmp
             name: temp-vol
-          - mountPath: /apiserver.local.config/certificates
-            name: certs
           - mountPath: /certs
             name: certificates
             readOnly: true
@@ -88,8 +90,6 @@ spec:
       volumes:
       - name: temp-vol
         emptyDir: {}
-      - name: certs
-        emptyDir: {}
       - name: certificates
         secret:
           defaultMode: 420

config/rbac/role.yaml

@@ -53,6 +53,16 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apiregistration.k8s.io
+  resources:
+  - apiservices
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/ginkgo/v2 v2.5.0
 	github.com/onsi/gomega v1.24.1
-	github.com/open-policy-agent/cert-controller v0.5.0
+	github.com/open-policy-agent/cert-controller v0.6.0
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
@@ -99,9 +99,6 @@ replace (
 	// https://nvd.nist.gov/vuln/detail/CVE-2022-1996
 	github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0+incompatible
 
-	// Needed for certificate generation till the PR is merged: https://github.com/open-policy-agent/cert-controller/pull/52
-	github.com/open-policy-agent/cert-controller => github.com/jorturfer/cert-controller v0.5.0
-
 	// https://avd.aquasec.com/nvd/2022/cve-2022-27191/
 	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
 

go.sum

@@ -589,8 +589,6 @@ github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqx
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
-github.com/jorturfer/cert-controller v0.5.0 h1:NuII7efZ14UsCn6a4vPbsQ+POvcmJ0S/UFPIkkXpRFw=
-github.com/jorturfer/cert-controller v0.5.0/go.mod h1:uOQW+2tMU51vSxy1Yt162oVUTMdqLuotC0aObQxrh6k=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
@@ -737,6 +735,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
 github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
+github.com/open-policy-agent/cert-controller v0.6.0 h1:HBhe1kS0GTk5dRHdklwgJKoGIctWisueIYnIYJu65Q0=
+github.com/open-policy-agent/cert-controller v0.6.0/go.mod h1:uOQW+2tMU51vSxy1Yt162oVUTMdqLuotC0aObQxrh6k=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=