Use proper certs for controller manager and scheduler

Don't rely on the self-signed ones. Add the loopback IP to the SAN list. Move the private getLoopbackIP function used by NLLB to k0s's internal net package, so that it can be used from multiple places. Also, configure the appropriate cipher suites and minimal TLS version. Signed-off-by: Tom Wieczorek <[email protected]>
twz123 · Aug 2, 2024 · 0280348 · 0280348
1 parent a4a9790
commit 0280348
Show file tree

Hide file tree

Showing 7 changed files with 172 additions and 47 deletions.
diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go
@@ -26,6 +26,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"slices"
+	"sync"
 	"syscall"
 	"time"
 
@@ -34,6 +35,7 @@ import (
 	"github.com/k0sproject/k0s/internal/pkg/dir"
 	"github.com/k0sproject/k0s/internal/pkg/file"
 	k0slog "github.com/k0sproject/k0s/internal/pkg/log"
+	internalnet "github.com/k0sproject/k0s/internal/pkg/net"
 	"github.com/k0sproject/k0s/internal/pkg/sysinfo"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/applier"
@@ -482,8 +484,16 @@ func (c *command) start(ctx context.Context) error {
 		clusterComponents.Add(ctx, controller.NewMetricServer(c.K0sVars, adminClientFactory))
 	}
 
+	loopbackIP := sync.OnceValue(func() net.IP {
+		ip, err := internalnet.LookupLoopbackIP(ctx)
+		if err != nil {
+			logrus.WithError(err).Warnf("Using %s as loopback address", ip)
+		}
+		return ip
+	})
+
 	if c.EnableMetricsScraper {
-		metrics, err := metrics.NewComponent(c.K0sVars, adminClientFactory, nodeConfig.Spec.Storage.Type)
+		metrics, err := metrics.NewComponent(c.K0sVars, loopbackIP(), adminClientFactory, nodeConfig.Spec.Storage.Type)
 		if err != nil {
 			return fmt.Errorf("failed to create metrics reconciler: %w", err)
 		}
@@ -525,9 +535,10 @@ func (c *command) start(ctx context.Context) error {
 
 	if !slices.Contains(c.DisableComponents, constant.KubeSchedulerComponentName) {
 		clusterComponents.Add(ctx, &controller.Scheduler{
-			LogLevel:   c.LogLevels.KubeScheduler,
-			K0sVars:    c.K0sVars,
-			SingleNode: c.SingleNode,
+			LogLevel:    c.LogLevels.KubeScheduler,
+			K0sVars:     c.K0sVars,
+			SingleNode:  c.SingleNode,
+			BindAddress: loopbackIP(),
 		})
 	}
 
@@ -536,6 +547,7 @@ func (c *command) start(ctx context.Context) error {
 			LogLevel:              c.LogLevels.KubeControllerManager,
 			K0sVars:               c.K0sVars,
 			SingleNode:            c.SingleNode,
+			BindAddress:           loopbackIP(),
 			ServiceClusterIPRange: nodeConfig.Spec.Network.BuildServiceCIDR(nodeConfig.Spec.API.Address),
 			ExtraArgs:             c.KubeControllerManagerExtraArgs,
 		})

diff --git a/internal/pkg/net/lookup.go b/internal/pkg/net/lookup.go
@@ -0,0 +1,54 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package net
+
+import (
+	"context"
+	"fmt"
+	"net"
+)
+
+// Resolves the preferred loopback IP address for this machine.
+//
+// Performs a DNS lookup for "localhost" and returns the first address that is
+// recognized as a loopback address. Returns an error if the DNS lookup fails
+// (e.g. due to context cancellation) or the lookup didn't return a loopback
+// address. In that case, a sane default loopback address is returned.
+//
+// Example usage:
+//
+//	ip, err := LookupLoopbackIP(ctx)
+//	if err != nil {
+//	    fmt.Fprintln(os.Stderr, "Failed to find loopback IP, falling back to", ip.String(), "-", err)
+//	}
+//	// ... use ip somehow
+func LookupLoopbackIP(ctx context.Context) (net.IP, error) {
+	localIPs, err := net.DefaultResolver.LookupIPAddr(ctx, "localhost")
+	if err != nil {
+		err = fmt.Errorf("failed to resolve localhost: %w", err)
+	} else {
+		for _, addr := range localIPs {
+			if addr.IP.IsLoopback() {
+				return addr.IP, nil
+			}
+		}
+
+		err = fmt.Errorf("no loopback IPs found for localhost: %v", localIPs)
+	}
+
+	return net.IP{127, 0, 0, 1}, err
+}
diff --git a/pkg/component/controller/controllermanager.go b/pkg/component/controller/controllermanager.go
@@ -19,34 +19,40 @@ package controller
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/k0sproject/k0s/internal/pkg/flags"
 	"github.com/k0sproject/k0s/internal/pkg/stringmap"
 	"github.com/k0sproject/k0s/internal/pkg/users"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/assets"
+	"github.com/k0sproject/k0s/pkg/certificate"
 	"github.com/k0sproject/k0s/pkg/component/manager"
 	"github.com/k0sproject/k0s/pkg/config"
 	"github.com/k0sproject/k0s/pkg/constant"
 	"github.com/k0sproject/k0s/pkg/supervisor"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/sirupsen/logrus"
 )
 
 // Manager implement the component interface to run kube scheduler
 type Manager struct {
 	K0sVars               *config.CfgVars
 	LogLevel              string
 	SingleNode            bool
+	BindAddress           net.IP
 	ServiceClusterIPRange string
 	ExtraArgs             string
 
 	supervisor     *supervisor.Supervisor
 	uid, gid       int
+	certificate    *certificate.Certificate
 	previousConfig stringmap.StringMap
 }
 
@@ -56,20 +62,47 @@ var _ manager.Component = (*Manager)(nil)
 var _ manager.Reconciler = (*Manager)(nil)
 
 // Init extracts the needed binaries
-func (a *Manager) Init(_ context.Context) error {
-	var err error
-	// controller manager running as api-server user as they both need access to same sa.key
-	a.uid, err = users.GetUID(constant.ApiserverUser)
-	if err != nil {
-		logrus.Warn("running kube-controller-manager as root: ", err)
+func (a *Manager) Init(ctx context.Context) error {
+	log := logrus.WithField("component", kubeControllerManagerComponent)
+
+	eg, ctx := errgroup.WithContext(ctx)
+	eg.Go(func() error {
+		return assets.Stage(a.K0sVars.BinDir, kubeControllerManagerComponent, constant.BinDirMode)
+	})
+
+	eg.Go(func() (err error) {
+		// controller manager running as api-server user as they both need access to same sa.key
+		a.uid, err = users.GetUID(constant.ApiserverUser)
+		if err != nil {
+			a.uid = 0
+			log.WithError(err).Warn("Running kube-controller-manager as root")
+		}
+		return nil
+	})
+
+	eg.Go(func() (err error) {
+		req := certificate.Request{
+			Name:      kubeControllerManagerComponent,
+			CN:        kubeControllerManagerComponent,
+			O:         "kubernetes",
+			Hostnames: []string{a.BindAddress.String()},
+		}
+
+		a.certificate, err = a.K0sVars.CertManager().EnsureCertificate(req, constant.ApiserverUser)
+		return err
+	})
+
+	if err := eg.Wait(); err != nil {
+		return err
 	}
 
 	// controller manager should be the only component that needs access to
 	// ca.key so let it own it.
-	if err := os.Chown(path.Join(a.K0sVars.CertRootDir, "ca.key"), a.uid, -1); err != nil && os.Geteuid() == 0 {
-		logrus.Warn("failed to change permissions for the ca.key: ", err)
+	if err := os.Chown(filepath.Join(a.K0sVars.CertRootDir, "ca.key"), a.uid, -1); err != nil && os.Geteuid() == 0 {
+		log.WithError(err).Warn("Failed to change permissions for the CA key file")
 	}
-	return assets.Stage(a.K0sVars.BinDir, kubeControllerManagerComponent, constant.BinDirMode)
+
+	return nil
 }
 
 // Run runs kube Manager
@@ -84,7 +117,11 @@ func (a *Manager) Reconcile(_ context.Context, clusterConfig *v1beta1.ClusterCon
 		"authentication-kubeconfig":        ccmAuthConf,
 		"authorization-kubeconfig":         ccmAuthConf,
 		"kubeconfig":                       ccmAuthConf,
-		"bind-address":                     "127.0.0.1",
+		"bind-address":                     a.BindAddress.String(),
+		"tls-cert-file":                    a.certificate.CertPath,
+		"tls-private-key-file":             a.certificate.KeyPath,
+		"tls-min-version":                  "VersionTLS12",
+		"tls-cipher-suites":                constant.AllowedTLS12CipherSuiteNames(),
 		"client-ca-file":                   path.Join(a.K0sVars.CertRootDir, "ca.crt"),
 		"cluster-signing-cert-file":        path.Join(a.K0sVars.CertRootDir, "ca.crt"),
 		"cluster-signing-key-file":         path.Join(a.K0sVars.CertRootDir, "ca.key"),

diff --git a/pkg/component/controller/metrics/component.go b/pkg/component/controller/metrics/component.go
@@ -22,6 +22,7 @@ import (
 	_ "embed"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -52,6 +53,7 @@ type Component struct {
 	log logrus.FieldLogger
 
 	hostname    string
+	loopbackIP  net.IP
 	K0sVars     *config.CfgVars
 	restClient  rest.Interface
 	storageType v1beta1.StorageType
@@ -65,7 +67,7 @@ var _ manager.Component = (*Component)(nil)
 var _ manager.Reconciler = (*Component)(nil)
 
 // NewComponent creates new Metrics reconciler
-func NewComponent(k0sVars *config.CfgVars, clientCF kubernetes.ClientFactoryInterface, storageType v1beta1.StorageType) (*Component, error) {
+func NewComponent(k0sVars *config.CfgVars, loopbackIP net.IP, clientCF kubernetes.ClientFactoryInterface, storageType v1beta1.StorageType) (*Component, error) {
 	hostname, err := os.Hostname()
 	if err != nil {
 		return nil, err
@@ -80,6 +82,7 @@ func NewComponent(k0sVars *config.CfgVars, clientCF kubernetes.ClientFactoryInte
 		log:         logrus.WithFields(logrus.Fields{"component": "metrics"}),
 		storageType: storageType,
 		hostname:    hostname,
+		loopbackIP:  loopbackIP,
 		K0sVars:     k0sVars,
 		restClient:  restClient,
 		jobs:        make(map[string]Scraper),
@@ -93,13 +96,13 @@ func (c *Component) Init(_ context.Context) error {
 	}
 
 	var j *job
-	j, err := c.newJob("https://localhost:10259/metrics")
+	j, err := c.newJob("https://" + net.JoinHostPort(c.loopbackIP.String(), "10259") + "/metrics")
 	if err != nil {
 		return err
 	}
 	c.jobs["kube-scheduler"] = j
 
-	j, err = c.newJob("https://localhost:10257/metrics")
+	j, err = c.newJob("https://" + net.JoinHostPort(c.loopbackIP.String(), "10257") + "/metrics")
 	if err != nil {
 		return err
 	}

diff --git a/pkg/component/controller/scheduler.go b/pkg/component/controller/scheduler.go
@@ -19,18 +19,22 @@ package controller
 import (
 	"context"
 	"fmt"
+	"net"
 	"path/filepath"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/k0sproject/k0s/internal/pkg/stringmap"
 	"github.com/k0sproject/k0s/internal/pkg/users"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/assets"
+	"github.com/k0sproject/k0s/pkg/certificate"
 	"github.com/k0sproject/k0s/pkg/component/manager"
 	"github.com/k0sproject/k0s/pkg/config"
 	"github.com/k0sproject/k0s/pkg/constant"
 	"github.com/k0sproject/k0s/pkg/supervisor"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/sirupsen/logrus"
 )
 
 // Scheduler implement the component interface to run kube scheduler
@@ -39,8 +43,10 @@ type Scheduler struct {
 	K0sVars        *config.CfgVars
 	LogLevel       string
 	SingleNode     bool
+	BindAddress    net.IP
 	supervisor     *supervisor.Supervisor
 	uid            int
+	certificate    *certificate.Certificate
 	previousConfig stringmap.StringMap
 }
 
@@ -50,13 +56,41 @@ var _ manager.Reconciler = (*Scheduler)(nil)
 const kubeSchedulerComponentName = "kube-scheduler"
 
 // Init extracts the needed binaries
-func (a *Scheduler) Init(_ context.Context) error {
-	var err error
-	a.uid, err = users.GetUID(constant.SchedulerUser)
-	if err != nil {
-		logrus.Warn("running kube-scheduler as root: ", err)
+func (a *Scheduler) Init(ctx context.Context) error {
+	log := logrus.WithField("component", kubeSchedulerComponentName)
+
+	eg, ctx := errgroup.WithContext(ctx)
+	eg.Go(func() error {
+		return assets.Stage(a.K0sVars.BinDir, kubeSchedulerComponentName, constant.BinDirMode)
+	})
+
+	eg.Go(func() (err error) {
+		a.uid, err = users.GetUID(constant.SchedulerUser)
+		if err != nil {
+			a.uid = 0
+			log.WithError(err).Warn("Running kube-scheduler as root")
+		}
+		return nil
+	})
+
+	eg.Go(func() (err error) {
+
+		req := certificate.Request{
+			Name:      kubeSchedulerComponentName,
+			CN:        kubeSchedulerComponentName,
+			O:         "kubernetes",
+			Hostnames: []string{a.BindAddress.String()},
+		}
+
+		a.certificate, err = a.K0sVars.CertManager().EnsureCertificate(req, constant.ApiserverUser)
+		return err
+	})
+
+	if err := eg.Wait(); err != nil {
+		return err
 	}
-	return assets.Stage(a.K0sVars.BinDir, kubeSchedulerComponentName, constant.BinDirMode)
+
+	return nil
 }
 
 // Run runs kube scheduler
@@ -82,7 +116,10 @@ func (a *Scheduler) Reconcile(_ context.Context, clusterConfig *v1beta1.ClusterC
 		"authentication-kubeconfig": schedulerAuthConf,
 		"authorization-kubeconfig":  schedulerAuthConf,
 		"kubeconfig":                schedulerAuthConf,
-		"bind-address":              "127.0.0.1",
+		"bind-address":              a.BindAddress.String(),
+		"tls-cert-file":             a.certificate.CertPath,
+		"tls-private-key-file":      a.certificate.KeyPath,
+		"tls-min-version":           "VersionTLS12",
 		"leader-elect":              fmt.Sprint(!a.SingleNode),
 		"profiling":                 "false",
 		"v":                         a.LogLevel,

diff --git a/pkg/component/worker/nllb/envoy.go b/pkg/component/worker/nllb/envoy.go
@@ -131,7 +131,7 @@ func (e *envoyProxy) start(ctx context.Context, profile workerconfig.Profile, ap
 		}
 	}()
 
-	loopbackIP, err := getLoopbackIP(ctx)
+	loopbackIP, err := k0snet.LookupLoopbackIP(ctx)
 	if err != nil {
 		if errors.Is(err, ctx.Err()) {
 			return err