Enable etcd-manager / etcd3 / etcd-tls in kops 1.12

dns-controller/pkg/dns/dnscontroller.go

@@ -320,6 +320,43 @@ func (c *DNSController) runOnce() error {
 	return nil
 }
 
+func (c *DNSController) RemoveRecordsImmediate(records []Record) error {
+	op, err := newDNSOp(c.zoneRules, c.dnsCache)
+	if err != nil {
+		return err
+	}
+
+	// Store a list of all the errors, so that one bad apple doesn't block every other request
+	var errors []error
+
+	for _, r := range records {
+		k := recordKey{
+			RecordType: r.RecordType,
+			FQDN:       r.FQDN,
+		}
+
+		err := op.deleteRecords(k)
+		if err != nil {
+			glog.Infof("error deleting records for %s: %v", k, err)
+			errors = append(errors, err)
+		}
+	}
+
+	for key, changeset := range op.changesets {
+		glog.V(2).Infof("applying DNS changeset for zone %s", key)
+		if err := changeset.Apply(); err != nil {
+			glog.Warningf("error applying DNS changeset for zone %s: %v", key, err)
+			errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err))
+		}
+	}
+
+	if len(errors) != 0 {
+		return errors[0]
+	}
+
+	return nil
+}
+
 // dnsOp manages a single dns change; we cache results and state for the duration of the operation
 type dnsOp struct {
 	dnsCache     *dnsCache

docs/etcd/manager.md

@@ -10,10 +10,6 @@ built into kops, but it addresses some limitations also:
 * allows etcd2 -> etcd3 upgrade (along with minor upgrades)
 * allows cluster resizing (e.g. going from 1 to 3 nodes)
 
-etcd-manager is currently in early alpha - it is undergoing testing. However, if
-you have a test cluster where you don't mind if it erases all your data, please
-do try it out and provide feedback.
-
 If using kubernetes >= 1.12 (which will not formally be supported until kops 1.12), note that etcd-manager will be used by default.  You can override this with the `cluster.spec.etcdClusters[*].provider=Legacy` override.  This can be specified:
 
 * as an argument to `kops create cluster`: `--overrides cluster.spec.etcdClusters[*].provider=Legacy`
@@ -24,8 +20,8 @@ If using kubernetes >= 1.12 (which will not formally be supported until kops 1.1
 
 ## How to use etcd-manager
 
-Reminder: etcd-manager is alpha, and may cause you to lose the data in your
-kubernetes cluster.
+Reminder: etcd-manager is alpha, and we encourage you to back up the data in
+your kubernetes cluster.
 
 To create a test cluster:
 ```bash
@@ -67,7 +63,7 @@ kops delete cluster example.k8s.local --yes
 ```
 
 You can also do this for existing clusters. though remember that this is still
-an early alpha, so don't use for clusters with important data.  You just run the
+young software, so please back up important cluster data first.  You just run the
 two `kops set cluster` commands.  Note that `kops set cluster` is just an easy
 command line way to set some fields in the cluster spec - if you're using a
 GitOps approach you can change the manifest files directly. You can also `kops

docs/releases/1.12-NOTES.md

@@ -0,0 +1,25 @@
+## Release notes for kops 1.12 series
+
+# Significant changes
+
+* kops 1.12 enables etcd-manager by default.  For kubernetes 1.12 (and later) we
+  default to etcd3.  We also enable TLS for etcd communications when using
+  etcd-manager.  More information is in the [etcd migration
+  documentation](https://github.com/kubernetes/kops/blob/master/docs/etcd3-migration.md)
+* Components are no longer allowed to interact with etcd directly.  Calico will
+  be switched to use CRDs instead of directly with etcd.  This is a disruptive
+  upgrade, please read the calico notes in the [etcd migration
+  documentation](https://github.com/kubernetes/kops/blob/master/docs/etcd3-migration.md)
+
+# Required Actions
+
+* Please back-up important data before upgrading, as the [etcd2 to etcd3
+  migration](https://github.com/kubernetes/kops/blob/master/docs/etcd3-migration.md)
+  is higher risk than most upgrades.
+* Note that the upgrade for Calico users is disruptive, because it requires
+  switching from direct-etcd-storage to CRD backed storage.
+
+# Full change list since 1.11.0 release
+
+(will be included with 1.12.0 beta releases)
+

nodeup/pkg/model/BUILD.bazel

@@ -67,6 +67,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
+        "//vendor/k8s.io/client-go/util/cert:go_default_library",
         "//vendor/k8s.io/kubernetes/pkg/util/mount:go_default_library",
     ],
 )

nodeup/pkg/model/context.go

@@ -287,6 +287,17 @@ func (c *NodeupModelContext) IsKubernetesGTE(version string) bool {
 	return util.IsKubernetesGTE(version, c.kubernetesVersion)
 }
 
+// UseEtcdManager checks if the etcd cluster has etcd-manager enabled
+func (c *NodeupModelContext) UseEtcdManager() bool {
+	for _, x := range c.Cluster.Spec.EtcdClusters {
+		if x.Provider == kops.EtcdProviderTypeManager {
+			return true
+		}
+	}
+
+	return false
+}
+
 // UseEtcdTLS checks if the etcd cluster has TLS enabled bool
 func (c *NodeupModelContext) UseEtcdTLS() bool {
 	// @note: because we enforce that 'both' have to be enabled for TLS we only need to check one here.

nodeup/pkg/model/convenience.go

@@ -119,6 +119,23 @@ func addHostPathMapping(pod *v1.Pod, container *v1.Container, name, path string)
 	return &container.VolumeMounts[len(container.VolumeMounts)-1]
 }
 
+// addHostPathVolume is shorthand for mapping a host path into a container
+func addHostPathVolume(pod *v1.Pod, container *v1.Container, hostPath v1.HostPathVolumeSource, volumeMount v1.VolumeMount) {
+	vol := v1.Volume{
+		Name: volumeMount.Name,
+		VolumeSource: v1.VolumeSource{
+			HostPath: &hostPath,
+		},
+	}
+
+	if volumeMount.MountPath == "" {
+		volumeMount.MountPath = hostPath.Path
+	}
+
+	pod.Spec.Volumes = append(pod.Spec.Volumes, vol)
+	container.VolumeMounts = append(container.VolumeMounts, volumeMount)
+}
+
 // convEtcdSettingsToMs converts etcd settings to a string rep of int milliseconds
 func convEtcdSettingsToMs(dur *metav1.Duration) string {
 	return strconv.FormatInt(dur.Nanoseconds()/1000000, 10)

nodeup/pkg/model/etcd_manager_tls.go

@@ -17,7 +17,15 @@ limitations under the License.
 package model
 
 import (
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	"github.com/golang/glog"
+	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
@@ -39,6 +47,9 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
 
 		keys := make(map[string]string)
 		keys["etcd-manager-ca"] = "etcd-manager-ca-" + k
+		keys["etcd-peers-ca"] = "etcd-peers-ca-" + k
+		// Because API server can only have a single client-cert, we need to share a client CA
+		keys["etcd-clients-ca"] = "etcd-clients-ca"
 
 		for fileName, keystoreName := range keys {
 			cert, err := b.KeyStore.FindCert(keystoreName)
@@ -59,5 +70,86 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
 		}
 	}
 
+	// We also dynamically generate the client keypair for apiserver
+	if err := b.buildKubeAPIServerKeypair(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *EtcdManagerTLSBuilder) buildKubeAPIServerKeypair() error {
+	etcdClientsCACertificate, err := b.KeyStore.FindCert("etcd-clients-ca")
+	if err != nil {
+		return err
+	}
+
+	etcdClientsCAPrivateKey, err := b.KeyStore.FindPrivateKey("etcd-clients-ca")
+	if err != nil {
+		return err
+	}
+
+	if etcdClientsCACertificate == nil {
+		glog.Errorf("unable to find etcd-clients-ca certificate, won't build key for apiserver")
+		return nil
+	}
+	if etcdClientsCAPrivateKey == nil {
+		glog.Errorf("unable to find etcd-clients-ca private key, won't build key for apiserver")
+		return nil
+	}
+
+	dir := "/etc/kubernetes/pki/kube-apiserver"
+
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("error creating directories %q: %v", dir, err)
+	}
+
+	{
+		p := filepath.Join(dir, "etcd-ca.crt")
+		certBytes := certutil.EncodeCertPEM(etcdClientsCACertificate.Certificate)
+		if err := ioutil.WriteFile(p, certBytes, 0644); err != nil {
+			return fmt.Errorf("error writing certificate key file %q: %v", p, err)
+		}
+	}
+
+	name := "etcd-client"
+
+	humanName := dir + "/" + name
+	privateKey, err := certutil.NewPrivateKey()
+	if err != nil {
+		return fmt.Errorf("unable to create private key %q: %v", humanName, err)
+	}
+	privateKeyBytes := certutil.EncodePrivateKeyPEM(privateKey)
+
+	certConfig := certutil.Config{
+		CommonName: "kube-apiserver",
+		Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+
+	signingKey, ok := etcdClientsCAPrivateKey.Key.(*rsa.PrivateKey)
+	if !ok {
+		return fmt.Errorf("etcd-clients-ca private key had unexpected type %T", etcdClientsCAPrivateKey.Key)
+	}
+
+	glog.Infof("signing certificate for %q", humanName)
+	cert, err := certutil.NewSignedCert(certConfig, privateKey, etcdClientsCACertificate.Certificate, signingKey)
+	if err != nil {
+		return fmt.Errorf("error signing certificate for %q: %v", humanName, err)
+	}
+
+	certBytes := certutil.EncodeCertPEM(cert)
+
+	p := filepath.Join(dir, name)
+	{
+		if err := ioutil.WriteFile(p+".crt", certBytes, 0644); err != nil {
+			return fmt.Errorf("error writing certificate key file %q: %v", p+".crt", err)
+		}
+	}
+
+	{
+		if err := ioutil.WriteFile(p+".key", privateKeyBytes, 0600); err != nil {
+			return fmt.Errorf("error writing private key file %q: %v", p+".key", err)
+		}
+	}
+
 	return nil
 }

nodeup/pkg/model/etcd_tls.go

@@ -32,7 +32,7 @@ var _ fi.ModelBuilder = &EtcdTLSBuilder{}
 // Build is responsible for performing setup for CNIs that need etcd TLS support
 func (b *EtcdTLSBuilder) Build(c *fi.ModelBuilderContext) error {
 	// @check if tls is enabled and if so, we need to download the client certificates
-	if b.UseEtcdTLS() {
+	if !b.UseEtcdManager() && b.UseEtcdTLS() {
 		name := "calico-client"
 		dirname := "calico"
 		ca := filepath.Join(dirname, "ca.pem")

nodeup/pkg/model/kube_apiserver.go

@@ -289,7 +289,14 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 		kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv")
 	}
 
-	if b.UseEtcdTLS() {
+	if b.UseEtcdManager() && b.UseEtcdTLS() {
+		basedir := "/etc/kubernetes/pki/kube-apiserver"
+		kubeAPIServer.EtcdCAFile = filepath.Join(basedir, "etcd-ca.crt")
+		kubeAPIServer.EtcdCertFile = filepath.Join(basedir, "etcd-client.crt")
+		kubeAPIServer.EtcdKeyFile = filepath.Join(basedir, "etcd-client.key")
+		kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"}
+		kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"}
+	} else if b.UseEtcdTLS() {
 		kubeAPIServer.EtcdCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 		kubeAPIServer.EtcdCertFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client.pem")
 		kubeAPIServer.EtcdKeyFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client-key.pem")
@@ -421,6 +428,19 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 
 	addHostPathMapping(pod, container, "logfile", "/var/log/kube-apiserver.log").ReadOnly = false
 
+	if b.UseEtcdManager() {
+		volumeType := v1.HostPathDirectoryOrCreate
+		addHostPathVolume(pod, container,
+			v1.HostPathVolumeSource{
+				Path: "/etc/kubernetes/pki/kube-apiserver",
+				Type: &volumeType,
+			},
+			v1.VolumeMount{
+				Name:     "pki",
+				ReadOnly: false,
+			})
+	}
+
 	// Add cloud config file if needed
 	if b.Cluster.Spec.CloudConfig != nil {
 		addHostPathMapping(pod, container, "cloudconfig", CloudConfigFilePath)

nodeup/pkg/model/protokube.go

@@ -68,7 +68,7 @@ func (t *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error {
 		})
 
 		// retrieve the etcd peer certificates and private keys from the keystore
-		if t.UseEtcdTLS() {
+		if !t.UseEtcdManager() && t.UseEtcdTLS() {
 			for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} {
 				if err := t.BuildCertificateTask(c, x, fmt.Sprintf("%s.pem", x)); err != nil {
 					return err
@@ -222,6 +222,10 @@ type ProtokubeFlags struct {
 
 	// ManageEtcd is true if protokube should manage etcd; being replaced by etcd-manager
 	ManageEtcd bool `json:"manageEtcd,omitempty" flag:"manage-etcd"`
+
+	// RemoveDNSNames allows us to remove dns records, so that they can be managed elsewhere
+	// We use it e.g. for the switch to etcd-manager
+	RemoveDNSNames string `json:"removeDNSNames,omitempty" flag:"remove-dns-names"`
 }
 
 // ProtokubeFlags is responsible for building the command line flags for protokube
@@ -364,6 +368,30 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) (*Protokube
 		f.ApplyTaints = fi.Bool(true)
 	}
 
+	// Remove DNS names if we're using etcd-manager
+	if !f.ManageEtcd {
+		var names []string
+
+		// Mirroring the logic used to construct DNS names in protokube/pkg/protokube/etcd_cluster.go
+		suffix := fi.StringValue(f.DNSInternalSuffix)
+		if !strings.HasPrefix(suffix, ".") {
+			suffix = "." + suffix
+		}
+
+		for _, c := range t.Cluster.Spec.EtcdClusters {
+			clusterName := "etcd-" + c.Name
+			if clusterName == "etcd-main" {
+				clusterName = "etcd"
+			}
+			for _, m := range c.Members {
+				name := clusterName + "-" + m.Name + suffix
+				names = append(names, name)
+			}
+		}
+
+		f.RemoveDNSNames = strings.Join(names, ",")
+	}
+
 	return f, nil
 }
 

pkg/model/components/etcd.go

@@ -18,6 +18,7 @@ package components
 
 import (
 	"fmt"
+	"strings"
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/upup/pkg/fi/loader"
@@ -74,6 +75,29 @@ func (b *EtcdOptionsBuilder) BuildOptions(o interface{}) error {
 				c.Version = DefaultEtcd2Version
 			}
 		}
+
+		// From 1.12, we enable TLS if we're running EtcdManager & etcd3
+		//
+		// (Moving to etcd3 is a disruptive upgrade, so we
+		// force TLS at the same time as we enable
+		// etcd-manager by default).
+		if c.Provider == kops.EtcdProviderTypeManager {
+			etcdV3 := true
+			version := c.Version
+			version = strings.TrimPrefix(version, "v")
+			if strings.HasPrefix(version, "2.") {
+				etcdV3 = false
+			} else if strings.HasPrefix(version, "3.") {
+				etcdV3 = true
+			} else {
+				return fmt.Errorf("unexpected etcd version %q", c.Version)
+			}
+
+			if b.IsKubernetesGTE("1.12.0") && etcdV3 {
+				c.EnableEtcdTLS = true
+				c.EnableTLSAuth = true
+			}
+		}
 	}
 
 	// Remap the well known images