From 929fa69932f979d9f77576aa04f1b8f254af5bfc Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Tue, 18 Jun 2024 11:11:37 -0600 Subject: [PATCH] UPSTREAM: 5859: Allow CA dirs to be specified beyond /custom/ca/ Signed-off-by: Joel Smith --- CHANGELOG.md | 1 + cmd/operator/main.go | 4 +++ pkg/util/certificates.go | 63 ++++++++++++++++++++--------------- pkg/util/certificates_test.go | 54 +++++++++++++++++++++++++----- 4 files changed, 87 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec13792ce3..f22c2b3a3a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ New deprecation(s): - **GCP Scalers**: Properly close the connection during the scaler cleaning process ([#5448](https://github.com/kedacore/keda/issues/5448)) - **GCP Scalers**: Restore previous time horizon to prevent querying issues ([#5429](https://github.com/kedacore/keda/issues/5429)) - **Prometheus Scaler**: Fix for missing AWS region from metadata ([#5419](https://github.com/kedacore/keda/issues/5419)) +- **General**: Add --ca-dir flag to KEDA operator to specify directories with CA certificates for scalers to authenticate TLS connections (defaults to /custom/ca) ([#5860](https://github.com/kedacore/keda/issues/5860)) ## v2.13.0 diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 945881595fb..7a15578a2dc 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -85,6 +85,7 @@ func main() { var k8sClusterDomain string var enableCertRotation bool var validatingWebhookName string + var caDirs []string pflag.BoolVar(&enablePrometheusMetrics, "enable-prometheus-metrics", true, "Enable the prometheus metric of keda-operator.") pflag.BoolVar(&enableOpenTelemetryMetrics, "enable-opentelemetry-metrics", false, "Enable the opentelemetry metric of keda-operator.") pflag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the prometheus metric endpoint binds to.") @@ -106,6 +107,7 @@ func main() { pflag.StringVar(&k8sClusterDomain, "k8s-cluster-domain", "cluster.local", "Kubernetes cluster domain. Defaults to cluster.local") pflag.BoolVar(&enableCertRotation, "enable-cert-rotation", false, "enable automatic generation and rotation of TLS certificates/keys") pflag.StringVar(&validatingWebhookName, "validating-webhook-name", "keda-admission", "ValidatingWebhookConfiguration name. Defaults to keda-admission") + pflag.StringArrayVar(&caDirs, "ca-dir", []string{"/custom/ca"}, "Directory with CA certificates for scalers to authenticate TLS connections. Can be specified multiple times. Defaults to /custom/ca") opts := zap.Options{} opts.BindFlags(flag.CommandLine) pflag.CommandLine.AddGoFlagSet(flag.CommandLine) @@ -300,6 +302,8 @@ func main() { close(certReady) } + kedautil.SetCACertDirs(caDirs) + grpcServer := metricsservice.NewGrpcServer(&scaledHandler, metricsServiceAddr, certDir, certReady) if err := mgr.Add(&grpcServer); err != nil { setupLog.Error(err, "unable to set up Metrics Service gRPC server") diff --git a/pkg/util/certificates.go b/pkg/util/certificates.go index 6d309165d46..580d09d7190 100644 --- a/pkg/util/certificates.go +++ b/pkg/util/certificates.go @@ -29,15 +29,23 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -const customCAPath = "/custom/ca" +const defaultCustomCAPath = "/custom/ca" var logger = logf.Log.WithName("certificates") var ( - rootCAs *x509.CertPool - rootCAsLock sync.Mutex + rootCAs *x509.CertPool + rootCAsLock sync.Mutex + customCAPaths = []string{defaultCustomCAPath} ) +// SetCACertDirs sets location(s) containing CA certificates which should be trusted for +// all future calls to CreateTLSClientConfig +func SetCACertDirs(caCertDirs []string) { + customCAPaths = caCertDirs + rootCAs = nil // force a reload on the next call to getRootCAs() +} + func getRootCAs() *x509.CertPool { rootCAsLock.Lock() defer rootCAsLock.Unlock() @@ -56,37 +64,38 @@ func getRootCAs() *x509.CertPool { logger.V(1).Info("system cert pool not available, using new cert pool instead") } } - if _, err := os.Stat(customCAPath); errors.Is(err, fs.ErrNotExist) { - logger.V(1).Info(fmt.Sprintf("the path %s doesn't exist, skipping custom CA registrations", customCAPath)) - return rootCAs - } - - files, err := os.ReadDir(customCAPath) - if err != nil { - logger.Error(err, fmt.Sprintf("unable to read %s", customCAPath)) - return rootCAs - } - - for _, file := range files { - filename := file.Name() - if file.IsDir() || strings.HasPrefix(filename, "..") { - logger.V(1).Info(fmt.Sprintf("%s isn't a valid certificate", filename)) - continue // Skip directories and special files + for _, customCAPath := range customCAPaths { + if _, err := os.Stat(customCAPath); errors.Is(err, fs.ErrNotExist) { + logger.V(1).Info(fmt.Sprintf("the path %s doesn't exist, skipping custom CA registrations", customCAPath)) + continue } - filePath := filepath.Join(customCAPath, filename) - certs, err := os.ReadFile(filePath) + files, err := os.ReadDir(customCAPath) if err != nil { - logger.Error(err, fmt.Sprintf("error reading %q", filename)) + logger.Error(err, fmt.Sprintf("unable to read %s", customCAPath)) continue } - if ok := rootCAs.AppendCertsFromPEM(certs); !ok { - logger.Error(fmt.Errorf("no certs appended"), "filename", filename) - continue + for _, file := range files { + filename := file.Name() + if file.IsDir() || strings.HasPrefix(filename, "..") { + logger.V(1).Info(fmt.Sprintf("%s isn't a valid certificate", filename)) + continue // Skip directories and special files + } + + filePath := filepath.Join(customCAPath, filename) + certs, err := os.ReadFile(filePath) + if err != nil { + logger.Error(err, fmt.Sprintf("error reading %q", filename)) + continue + } + + if ok := rootCAs.AppendCertsFromPEM(certs); !ok { + logger.Error(fmt.Errorf("no certs appended"), "filename", filename) + continue + } + logger.V(1).Info(fmt.Sprintf("the certificate %s has been added to the pool", filename)) } - logger.V(1).Info(fmt.Sprintf("the certificate %s has been added to the pool", filename)) } - return rootCAs } diff --git a/pkg/util/certificates_test.go b/pkg/util/certificates_test.go index 221edaa65d8..60314f36b2e 100644 --- a/pkg/util/certificates_test.go +++ b/pkg/util/certificates_test.go @@ -34,19 +34,24 @@ import ( ) var ( - caCrtPath = path.Join(customCAPath, "ca.crt") - certCommonName = "test-cert" + certCommonName = "test-cert" + certCommonName2 = "test-cert2" ) func TestCustomCAsAreRegistered(t *testing.T) { - defer os.Remove(caCrtPath) - generateCA(t) + customCAPath, err := os.MkdirTemp("", "test-ca-certdir-") + require.NoErrorf(t, err, "error creating temporary certs dir - %s", err) + defer os.RemoveAll(customCAPath) + + generateCA(t, certCommonName, customCAPath) + + SetCACertDirs([]string{customCAPath}) rootCAs := getRootCAs() //nolint:staticcheck // func (s *CertPool) Subjects was deprecated if s was returned by SystemCertPool, Subjects subjects := rootCAs.Subjects() var rdnSequence pkix.RDNSequence - _, err := asn1.Unmarshal(subjects[len(subjects)-1], &rdnSequence) + _, err = asn1.Unmarshal(subjects[len(subjects)-1], &rdnSequence) if err != nil { t.Fatal("could not unmarshal der formatted subject") } @@ -56,8 +61,41 @@ func TestCustomCAsAreRegistered(t *testing.T) { assert.Equal(t, certCommonName, name.CommonName, "certificate not found") } -func generateCA(t *testing.T) { - err := os.MkdirAll(customCAPath, os.ModePerm) +func TestMultipleCustomCADirs(t *testing.T) { + customCAPath, err := os.MkdirTemp("", "test-ca-certdir-") + require.NoErrorf(t, err, "error creating temporary certs dir - %s", err) + defer os.RemoveAll(customCAPath) + customCAPath2, err := os.MkdirTemp("", "test-ca-certdir2-") + require.NoErrorf(t, err, "error creating temporary certs dir - %s", err) + defer os.RemoveAll(customCAPath2) + + generateCA(t, certCommonName, customCAPath) + generateCA(t, certCommonName2, customCAPath2) + SetCACertDirs([]string{customCAPath, customCAPath2}) + + rootCAs := getRootCAs() + //nolint:staticcheck // func (s *CertPool) Subjects was deprecated if s was returned by SystemCertPool, Subjects + subjects := rootCAs.Subjects() + var rdnSequence pkix.RDNSequence + for i := 0; i < 2; i++ { + _, err = asn1.Unmarshal(subjects[len(subjects)-1-i], &rdnSequence) + if err != nil { + t.Fatal("could not unmarshal der formatted subject") + } + var name pkix.Name + name.FillFromRDNSequence(&rdnSequence) + + if i == 1 { + assert.Equal(t, certCommonName, name.CommonName, "certificate not found") + } else { + assert.Equal(t, certCommonName2, name.CommonName, "certificate not found") + } + } +} + +func generateCA(t *testing.T, cn, dir string) { + err := os.MkdirAll(dir, os.ModePerm) + caCrtPath := path.Join(dir, "ca.crt") require.NoErrorf(t, err, "error generating the custom ca folder - %s", err) ca := &x509.Certificate{ @@ -69,7 +107,7 @@ func generateCA(t *testing.T) { Locality: []string{"San Francisco"}, StreetAddress: []string{"Golden Gate Bridge"}, PostalCode: []string{"94016"}, - CommonName: certCommonName, + CommonName: cn, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0),