diff --git a/cmd/kops/create_cluster_integration_test.go b/cmd/kops/create_cluster_integration_test.go index b0947c08fd808..58087834d80df 100644 --- a/cmd/kops/create_cluster_integration_test.go +++ b/cmd/kops/create_cluster_integration_test.go @@ -19,6 +19,7 @@ package main import ( "bytes" "io/ioutil" + "os" "path" "strings" "testing" @@ -254,12 +255,27 @@ func runCreateClusterIntegrationTest(t *testing.T, srcDir string, version string actualYAML := strings.Join(yamlAll, "\n\n---\n\n") if actualYAML != expectedYAML { + p := path.Join(srcDir, expectedClusterPath) + + if os.Getenv("HACK_UPDATE_EXPECTED_IN_PLACE") != "" { + t.Logf("HACK_UPDATE_EXPECTED_IN_PLACE: writing expected output %s", p) + + // Format nicely - keep git happy + s := actualYAML + s = strings.TrimSpace(s) + s = s + "\n" + + if err := ioutil.WriteFile(p, []byte(s), 0644); err != nil { + t.Errorf("error writing expected output %s: %v", p, err) + } + } + glog.Infof("Actual YAML:\n%s\n", actualYAML) diffString := diff.FormatDiff(expectedYAML, actualYAML) t.Logf("diff:\n%s\n", diffString) - t.Fatalf("YAML differed from expected (%s)", path.Join(srcDir, expectedClusterPath)) + t.Errorf("YAML differed from expected (%s)", p) } } diff --git a/docs/etcd/manager.md b/docs/etcd/manager.md index 6df77a5324aca..d24f9e35b84da 100644 --- a/docs/etcd/manager.md +++ b/docs/etcd/manager.md @@ -14,6 +14,14 @@ etcd-manager is currently in early alpha - it is undergoing testing. However, if you have a test cluster where you don't mind if it erases all your data, please do try it out and provide feedback. +If using kubernetes >= 1.12 (which will not formally be supported until kops 1.12), note that etcd-manager will be used by default. You can override this with the `cluster.spec.etcdClusters[*].provider=Legacy` override. This can be specified: + +* as an argument to `kops create cluster`: `--overrides cluster.spec.etcdClusters[*].provider=Legacy` +* on an existing cluster with `kops set cluster cluster.spec.etcdClusters[*].provider=Legacy` +* by setting the field using `kops edit` or `kops replace`, manually making the same change as `kops set cluster ...` + +(Note you will probably have to `export KOPS_FEATURE_FLAGS=SpecOverrideFlag`) + ## How to use etcd-manager Reminder: etcd-manager is alpha, and may cause you to lose the data in your diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 48e47a1d7768c..580889f4c7d4f 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -354,10 +354,21 @@ type ExternalDNSConfig struct { WatchNamespace string `json:"watchNamespace,omitempty"` } +// EtcdProviderType describes etcd cluster provisioning types (Standalone, Manager) +type EtcdProviderType string + +const ( + EtcdProviderTypeManager EtcdProviderType = "Manager" + EtcdProviderTypeLegacy EtcdProviderType = "Legacy" +) + // EtcdClusterSpec is the etcd cluster specification type EtcdClusterSpec struct { // Name is the name of the etcd cluster (main, events etc) Name string `json:"name,omitempty"` + // Provider is the provider used to run etcd: standalone, manager. + // We default to manager for kubernetes 1.11 or if the manager is configured; otherwise standalone. + Provider EtcdProviderType `json:"provider,omitempty"` // Members stores the configurations for each member of the cluster (including the data volume) Members []*EtcdMemberSpec `json:"etcdMembers,omitempty"` // EnableEtcdTLS indicates the etcd service should use TLS between peers and clients diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index da42baee79164..b5593dadb5a97 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -353,10 +353,21 @@ type ExternalDNSConfig struct { WatchNamespace string `json:"watchNamespace,omitempty"` } +// EtcdProviderType describes etcd cluster provisioning types (Standalone, Manager) +type EtcdProviderType string + +const ( + EtcdProviderTypeManager EtcdProviderType = "Manager" + EtcdProviderTypeLegacy EtcdProviderType = "Legacy" +) + // EtcdClusterSpec is the etcd cluster specification type EtcdClusterSpec struct { // Name is the name of the etcd cluster (main, events etc) Name string `json:"name,omitempty"` + // Provider is the provider used to run etcd: standalone, manager. + // We default to manager for kubernetes 1.11 or if the manager is configured; otherwise standalone. + Provider EtcdProviderType `json:"provider,omitempty"` // Members stores the configurations for each member of the cluster (including the data volume) Members []*EtcdMemberSpec `json:"etcdMembers,omitempty"` // EnableTLSAuth indicates client and peer TLS auth should be enforced diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index ca87ea5e5a44c..ac7c2d71b6972 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1486,6 +1486,7 @@ func Convert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec(in *kops.EtcdBackupS func autoConvert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error { out.Name = in.Name + out.Provider = kops.EtcdProviderType(in.Provider) if in.Members != nil { in, out := &in.Members, &out.Members *out = make([]*kops.EtcdMemberSpec, len(*in)) @@ -1532,6 +1533,7 @@ func Convert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpe func autoConvert_kops_EtcdClusterSpec_To_v1alpha1_EtcdClusterSpec(in *kops.EtcdClusterSpec, out *EtcdClusterSpec, s conversion.Scope) error { out.Name = in.Name + out.Provider = EtcdProviderType(in.Provider) if in.Members != nil { in, out := &in.Members, &out.Members *out = make([]*EtcdMemberSpec, len(*in)) diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index dd6ece0f916c3..918b5338713ba 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -354,10 +354,21 @@ type ExternalDNSConfig struct { WatchNamespace string `json:"watchNamespace,omitempty"` } +// EtcdProviderType describes etcd cluster provisioning types (Standalone, Manager) +type EtcdProviderType string + +const ( + EtcdProviderTypeManager EtcdProviderType = "Manager" + EtcdProviderTypeLegacy EtcdProviderType = "Legacy" +) + // EtcdClusterSpec is the etcd cluster specification type EtcdClusterSpec struct { // Name is the name of the etcd cluster (main, events etc) Name string `json:"name,omitempty"` + // Provider is the provider used to run etcd: standalone, manager. + // We default to manager for kubernetes 1.11 or if the manager is configured; otherwise standalone. + Provider EtcdProviderType `json:"provider,omitempty"` // Members stores the configurations for each member of the cluster (including the data volume) Members []*EtcdMemberSpec `json:"etcdMembers,omitempty"` // EnableEtcdTLS indicates the etcd service should use TLS between peers and clients diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 35fd664b97778..e6a38798f0955 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1587,6 +1587,7 @@ func Convert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec(in *kops.EtcdBackupS func autoConvert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error { out.Name = in.Name + out.Provider = kops.EtcdProviderType(in.Provider) if in.Members != nil { in, out := &in.Members, &out.Members *out = make([]*kops.EtcdMemberSpec, len(*in)) @@ -1633,6 +1634,7 @@ func Convert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpe func autoConvert_kops_EtcdClusterSpec_To_v1alpha2_EtcdClusterSpec(in *kops.EtcdClusterSpec, out *EtcdClusterSpec, s conversion.Scope) error { out.Name = in.Name + out.Provider = EtcdProviderType(in.Provider) if in.Members != nil { in, out := &in.Members, &out.Members *out = make([]*EtcdMemberSpec, len(*in)) diff --git a/pkg/apis/kops/validation/cluster.go b/pkg/apis/kops/validation/cluster.go index 48be77e5dcc72..44e36be697be1 100644 --- a/pkg/apis/kops/validation/cluster.go +++ b/pkg/apis/kops/validation/cluster.go @@ -41,7 +41,7 @@ func ValidateClusterUpdate(obj *kops.Cluster, status *kops.ClusterStatus, old *k } for k, newCluster := range newClusters { - fp := field.NewPath("Spec", "EtcdClusters").Key(k) + fp := field.NewPath("spec", "etcdClusters").Key(k) oldCluster := oldClusters[k] allErrs = append(allErrs, validateEtcdClusterUpdate(fp, newCluster, status, oldCluster)...) @@ -49,7 +49,7 @@ func ValidateClusterUpdate(obj *kops.Cluster, status *kops.ClusterStatus, old *k for k := range oldClusters { newCluster := newClusters[k] if newCluster == nil { - fp := field.NewPath("Spec", "EtcdClusters").Key(k) + fp := field.NewPath("spec", "etcdClusters").Key(k) allErrs = append(allErrs, field.Forbidden(fp, "EtcdClusters cannot be removed")) } } diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 0bcdabe336f30..1f56294407f61 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -36,7 +36,7 @@ import ( // ValidateCluster is responsible for checking the validity of the Cluster spec func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { - fieldSpec := field.NewPath("Spec") + fieldSpec := field.NewPath("spec") var err error // kubernetesRelease is the version with only major & minor fields @@ -573,7 +573,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { return field.Required(fieldEtcdClusters, "") } for i, x := range c.Spec.EtcdClusters { - if err := validateEtcdClusterSpec(x, fieldEtcdClusters.Index(i)); err != nil { + if err := validateEtcdClusterSpecLegacy(x, fieldEtcdClusters.Index(i)); err != nil { return err } } @@ -633,8 +633,8 @@ func validateSubnetCIDR(networkCIDR *net.IPNet, additionalNetworkCIDRs []*net.IP return false } -// validateEtcdClusterSpec is responsible for validating the etcd cluster spec -func validateEtcdClusterSpec(spec *kops.EtcdClusterSpec, fieldPath *field.Path) *field.Error { +// validateEtcdClusterSpecLegacy is responsible for validating the etcd cluster spec +func validateEtcdClusterSpecLegacy(spec *kops.EtcdClusterSpec, fieldPath *field.Path) *field.Error { if spec.Name == "" { return field.Required(fieldPath.Child("Name"), "EtcdCluster did not have name") } @@ -697,7 +697,7 @@ func validateEtcdVersion(spec *kops.EtcdClusterSpec, fieldPath *field.Path, mini version := spec.Version if spec.Version == "" { - version = components.DefaultEtcdVersion + version = components.DefaultEtcd2Version } sem, err := semver.Parse(strings.TrimPrefix(version, "v")) diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index c262578b6619f..68831907296cd 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -103,6 +103,13 @@ func validateClusterSpec(spec *kops.ClusterSpec, fieldPath *field.Path) field.Er } } + // EtcdClusters + { + for i, etcdCluster := range spec.EtcdClusters { + allErrs = append(allErrs, validateEtcdClusterSpec(etcdCluster, fieldPath.Child("etcdClusters").Index(i))...) + } + } + return allErrs } @@ -315,3 +322,22 @@ func validateAdditionalPolicy(role string, policy string, fldPath *field.Path) f return errs } + +func validateEtcdClusterSpec(spec *kops.EtcdClusterSpec, fieldPath *field.Path) field.ErrorList { + errs := field.ErrorList{} + + switch spec.Provider { + case kops.EtcdProviderTypeManager: + // ok + case kops.EtcdProviderTypeLegacy: + // ok + + case "": + // blank means that the user accepts the recommendation + + default: + errs = append(errs, field.Invalid(fieldPath.Child("provider"), spec.Provider, "Provider must be Manager or Legacy")) + } + + return errs +} diff --git a/pkg/commands/set_cluster.go b/pkg/commands/set_cluster.go index 8a12214178393..a45b283b52e85 100644 --- a/pkg/commands/set_cluster.go +++ b/pkg/commands/set_cluster.go @@ -88,6 +88,10 @@ func SetClusterFields(fields []string, cluster *api.Cluster, instanceGroups []*a for _, c := range cluster.Spec.EtcdClusters { c.Version = kv[1] } + case "cluster.spec.etcdClusters[*].provider": + for _, etcd := range cluster.Spec.EtcdClusters { + etcd.Provider = api.EtcdProviderType(kv[1]) + } case "cluster.spec.etcdClusters[*].manager.image": for _, etcd := range cluster.Spec.EtcdClusters { if etcd.Manager == nil { diff --git a/pkg/model/components/etcd.go b/pkg/model/components/etcd.go index 652c2a8789d59..016beff0909d2 100644 --- a/pkg/model/components/etcd.go +++ b/pkg/model/components/etcd.go @@ -27,80 +27,87 @@ const DefaultBackupImage = "kopeio/etcd-backup:1.0.20180220" // EtcdOptionsBuilder adds options for etcd to the model type EtcdOptionsBuilder struct { - Context *OptionsContext + *OptionsContext } var _ loader.OptionsBuilder = &EtcdOptionsBuilder{} -const DefaultEtcdVersion = "2.2.1" +const DefaultEtcd2Version = "2.2.1" // BuildOptions is responsible for filling in the defaults for the etcd cluster model func (b *EtcdOptionsBuilder) BuildOptions(o interface{}) error { spec := o.(*kops.ClusterSpec) for _, c := range spec.EtcdClusters { + if c.Provider == "" { + if b.IsKubernetesGTE("1.12") { + c.Provider = kops.EtcdProviderTypeManager + } else if c.Manager != nil { + c.Provider = kops.EtcdProviderTypeManager + } else { + c.Provider = kops.EtcdProviderTypeLegacy + } + } + // Ensure the version is set - // @TODO once we have a way of detecting a 'new' cluster we can default all clusters to v3 - if c.Version == "" { - c.Version = DefaultEtcdVersion + if c.Version == "" && c.Provider == kops.EtcdProviderTypeLegacy { + // Even if in legacy mode, etcd version 2 is unsupported as of k8s 1.13 + if b.IsKubernetesGTE("1.13") { + c.Version = "3.2.24" + } else { + c.Version = DefaultEtcd2Version + } } - useEtcdManager := false - if c.Manager != nil { - useEtcdManager = true + if c.Version == "" && c.Provider == kops.EtcdProviderTypeManager { + // From 1.11, we run the k8s-recommended versions of etcd when using the manager + if b.IsKubernetesGTE("1.11") { + // 1.11 originally recommended 3.2.18, but there was an advisory to update to 3.2.24 + c.Version = "3.2.24" + } else { + c.Version = DefaultEtcd2Version + } } + } + + // Remap the well known images + for _, c := range spec.EtcdClusters { + + // We remap the etcd manager image when we build the manifest, + // but we need to map the standalone images here because protokube launches them + + if c.Provider == kops.EtcdProviderTypeLegacy { - // remap etcd image - { - image := c.Image - if image == "" { - if !useEtcdManager { + // remap etcd image + { + image := c.Image + if image == "" { image = fmt.Sprintf("k8s.gcr.io/etcd:%s", c.Version) } - } - if image != "" { - image, err := b.Context.AssetBuilder.RemapImage(image) - if err != nil { - return fmt.Errorf("unable to remap container %q: %v", image, err) + if image != "" { + image, err := b.AssetBuilder.RemapImage(image) + if err != nil { + return fmt.Errorf("unable to remap container %q: %v", image, err) + } + c.Image = image } - c.Image = image } - } - // remap backup manager image - if c.Backups != nil { - image := c.Backups.Image - if image == "" { - if !useEtcdManager { + // remap backup manager image + if c.Backups != nil { + image := c.Backups.Image + if image == "" { image = fmt.Sprintf(DefaultBackupImage) } - } - - if image != "" { - image, err := b.Context.AssetBuilder.RemapImage(image) - if err != nil { - return fmt.Errorf("unable to remap container %q: %v", image, err) - } - c.Backups.Image = image - } - } - - // remap etcd manager image - if useEtcdManager { - image := c.Manager.Image - if image == "" { - // We can make this easier later - maybe put it into the channel? Or an addon? - //image = fmt.Sprintf(DefaultManagerImage) - return fmt.Errorf("EtcdManager.Image must be specified (currently)") - } - if image != "" { - image, err := b.Context.AssetBuilder.RemapImage(image) - if err != nil { - return fmt.Errorf("unable to remap container %q: %v", image, err) + if image != "" { + image, err := b.AssetBuilder.RemapImage(image) + if err != nil { + return fmt.Errorf("unable to remap container %q: %v", image, err) + } + c.Backups.Image = image } - c.Manager.Image = image } } } diff --git a/pkg/model/components/etcdmanager/BUILD.bazel b/pkg/model/components/etcdmanager/BUILD.bazel index 92ec81ab4cf3c..a53ac2a074331 100644 --- a/pkg/model/components/etcdmanager/BUILD.bazel +++ b/pkg/model/components/etcdmanager/BUILD.bazel @@ -24,15 +24,18 @@ go_library( "//upup/pkg/fi/fitasks:go_default_library", "//upup/pkg/fi/loader:go_default_library", "//util/pkg/exec:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", ], ) go_test( name = "go_default_test", srcs = ["model_test.go"], - data = glob(["tests/**"]), #keep embed = [":go_default_library"], deps = [ "//pkg/assets:go_default_library", diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index f799d7b15c12e..c51d8422102db 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -17,12 +17,19 @@ limitations under the License. package etcdmanager import ( + "bytes" "encoding/json" "fmt" + "io" + "os" "strings" + "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/yaml" + scheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/dns" @@ -51,7 +58,7 @@ var _ fi.ModelBuilder = &EtcdManagerBuilder{} // Build creates the tasks func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { for _, etcdCluster := range b.Cluster.Spec.EtcdClusters { - if etcdCluster.Manager == nil { + if etcdCluster.Provider != kops.EtcdProviderTypeManager { continue } @@ -118,16 +125,114 @@ func (b *EtcdManagerBuilder) buildManifest(etcdCluster *kops.EtcdClusterSpec) (* return b.buildPod(etcdCluster) } -// BuildEtcdManifest creates the pod spec, based on the etcd cluster +// parseManifest parses a set of objects from a []byte +func parseManifest(data []byte) ([]runtime.Object, error) { + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + deser := scheme.Codecs.UniversalDeserializer() + + var objects []runtime.Object + + for { + ext := runtime.RawExtension{} + if err := decoder.Decode(&ext); err != nil { + if err == io.EOF { + break + } + fmt.Fprintf(os.Stderr, "%s", string(data)) + glog.Infof("manifest: %s", string(data)) + return nil, fmt.Errorf("error parsing manifest: %v", err) + } + + obj, _, err := deser.Decode([]byte(ext.Raw), nil, nil) + if err != nil { + return nil, fmt.Errorf("error parsing object in manifest: %v", err) + } + + objects = append(objects, obj) + } + + return objects, nil +} + +// Until we introduce the bundle, we hard-code the manifest +var defaultManifest = ` +apiVersion: v1 +kind: Pod +metadata: + name: etcd-manager + namespace: kube-system +spec: + containers: + - image: kopeio/etcd-manager:1.0.20180729 + name: etcd-manager + resources: + requests: + cpu: 100m + # TODO: Would be nice to reduce these permissions; needed for volume mounting + securityContext: + privileged: true + volumeMounts: + # TODO: Would be nice to scope this more tightly, but needed for volume mounting + - mountPath: /rootfs + name: rootfs + # We write artificial hostnames into etc hosts for the etcd nodes, so they have stable names + - mountPath: /etc/hosts + name: hosts + hostNetwork: true + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /etc/hosts + type: File + name: hosts +` + +// buildPod creates the pod spec, based on the EtcdClusterSpec func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Pod, error) { - image := etcdCluster.Manager.Image + var pod *v1.Pod + var container *v1.Container + + var manifest []byte + + // TODO: pull from bundle + bundle := "(embedded etcd manifest)" + manifest = []byte(defaultManifest) + { - remapped, err := b.AssetBuilder.RemapImage(image) + objects, err := parseManifest(manifest) if err != nil { - return nil, fmt.Errorf("unable to remap container %q: %v", image, err) + return nil, err + } + if len(objects) != 1 { + return nil, fmt.Errorf("expected exactly one object in manifest %s, found %d", bundle, len(objects)) + } + if podObject, ok := objects[0].(*v1.Pod); !ok { + return nil, fmt.Errorf("expected v1.Pod object in manifest %s, found %T", bundle, objects[0]) } else { - image = remapped + pod = podObject + } + + if len(pod.Spec.Containers) != 1 { + return nil, fmt.Errorf("expected exactly one container in etcd-manager Pod, found %d", len(pod.Spec.Containers)) + } + container = &pod.Spec.Containers[0] + + if etcdCluster.Manager.Image != "" { + glog.Warningf("overloading image in manifest %s with images %s", bundle, etcdCluster.Manager.Image) + container.Image = etcdCluster.Manager.Image + } + } + + // Remap image via AssetBuilder + { + remapped, err := b.AssetBuilder.RemapImage(container.Image) + if err != nil { + return nil, fmt.Errorf("unable to remap container image %q: %v", container.Image, err) } + container.Image = remapped } isTLS := etcdCluster.EnableEtcdTLS @@ -142,7 +247,11 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po backupStore = etcdCluster.Backups.BackupStore } - podName := "etcd-manager-" + etcdCluster.Name + pod.Name = "etcd-manager-" + etcdCluster.Name + if pod.Labels == nil { + pod.Labels = make(map[string]string) + } + pod.Labels["k8s-app"] = pod.Name // TODO: Use a socket file for the quarantine port quarantinedClientPort := 3994 @@ -188,7 +297,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po } logFile := "/var/log/" + name + ".log" - config := &EtcdManagerConfig{ + config := &config{ Containerized: true, ClusterName: clusterName, BackupStore: backupStore, @@ -198,8 +307,6 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po config.LogVerbosity = 8 - var envs []v1.EnvVar - { // @check if we are using TLS scheme := "http" @@ -259,30 +366,14 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po return nil, err } - pod := &v1.Pod{} - pod.APIVersion = "v1" - pod.Kind = "Pod" - pod.Name = podName - pod.Namespace = "kube-system" - pod.Labels = map[string]string{"k8s-app": podName} - pod.Spec.HostNetwork = true - { - container := &v1.Container{ - Name: "etcd-manager", - Image: image, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: cpuRequest, - }, - }, - Command: exec.WithTee("/etcd-manager", args, "/var/log/etcd.log"), - Env: envs, - } + container.Command = exec.WithTee("/etcd-manager", args, "/var/log/etcd.log") - // TODO: Reduce these permissions (they are needed for volume mounting) - container.SecurityContext = &v1.SecurityContext{ - Privileged: fi.Bool(true), + // TODO: Should we try to incorporate the resources in the manifest? + container.Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: cpuRequest, + }, } // TODO: Use helper function here @@ -291,13 +382,6 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po MountPath: "/var/log/etcd.log", ReadOnly: false, }) - - // TODO: Would be nice to narrow this mount - container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{ - Name: "rootfs", - MountPath: "/rootfs", - ReadOnly: false, - }) hostPathFileOrCreate := v1.HostPathFileOrCreate pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ Name: "varlogetcd", @@ -309,24 +393,9 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po }, }) - hostPathDirectory := v1.HostPathDirectory - pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ - Name: "rootfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/", - Type: &hostPathDirectory, - }, - }, - }) - - kubemanifest.MapEtcHosts(pod, container, false) - if isTLS { return nil, fmt.Errorf("TLS not supported for etcd-manager") } - - pod.Spec.Containers = append(pod.Spec.Containers, *container) } kubemanifest.MarkPodAsCritical(pod) @@ -334,8 +403,8 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po return pod, nil } -// EtcdManagerConfig are the flags for etcd-manager -type EtcdManagerConfig struct { +// config defines the flags for etcd-manager +type config struct { // LogVerbosity sets the log verbosity level LogVerbosity int `flag:"v"` diff --git a/pkg/model/components/etcdmanager/options.go b/pkg/model/components/etcdmanager/options.go index 9fd0cc3fe96f9..29487d28f0be1 100644 --- a/pkg/model/components/etcdmanager/options.go +++ b/pkg/model/components/etcdmanager/options.go @@ -25,27 +25,38 @@ import ( // EtcdManagerOptionsBuilder adds options for the etcd-manager to the model. type EtcdManagerOptionsBuilder struct { - Context *components.OptionsContext + *components.OptionsContext } var _ loader.OptionsBuilder = &EtcdManagerOptionsBuilder{} -// BuildOptions generates the configurations used to create kubernetes controller manager manifest +// BuildOptions generates the configurations used to create etcd manager manifest func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec := o.(*kops.ClusterSpec) for _, etcdCluster := range clusterSpec.EtcdClusters { - if etcdCluster.Version == "" { - etcdCluster.Version = "2.2.1" + if etcdCluster.Provider != kops.EtcdProviderTypeManager { + continue } - if etcdCluster.Manager != nil { - if etcdCluster.Backups == nil { - etcdCluster.Backups = &kops.EtcdBackupSpec{} - } - if etcdCluster.Backups.BackupStore == "" { - base := clusterSpec.ConfigBase - etcdCluster.Backups.BackupStore = urls.Join(base, "backups", "etcd", etcdCluster.Name) + if etcdCluster.Manager == nil { + etcdCluster.Manager = &kops.EtcdManagerSpec{} + } + + if etcdCluster.Backups == nil { + etcdCluster.Backups = &kops.EtcdBackupSpec{} + } + if etcdCluster.Backups.BackupStore == "" { + base := clusterSpec.ConfigBase + etcdCluster.Backups.BackupStore = urls.Join(base, "backups", "etcd", etcdCluster.Name) + } + + if etcdCluster.Version == "" { + if b.IsKubernetesGTE("1.11") { + etcdCluster.Version = "3.2.18" + } else { + // Preserve existing default etcd version + etcdCluster.Version = "2.2.1" } } } diff --git a/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml b/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml index a6bfa70a682d0..f5c87ff35a669 100644 --- a/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml +++ b/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml @@ -16,17 +16,13 @@ spec: name: main backups: backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main - manager: - image: kopeio/etcd-manager:latest - etcdMembers: - instanceGroup: master-us-test-1a name: us-test-1a name: events backups: backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events - manager: - image: kopeio/etcd-manager:latest - kubernetesVersion: v1.8.0 + kubernetesVersion: v1.11.0 masterInternalName: api.internal.minimal.example.com masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml index 0294c44e55a2b..8aebee69a80c0 100644 --- a/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml +++ b/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml @@ -25,7 +25,7 @@ spec: iam: allowContainerRegistry: true legacy: false - kubernetesVersion: v1.4.8 + kubernetesVersion: v1.12.0 masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 networking: @@ -50,7 +50,7 @@ metadata: kops.k8s.io/cluster: minimal.example.com name: master-us-test-1a spec: - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2017-07-28 + image: kope.io/k8s-1.9-debian-jessie-amd64-hvm-ebs-2018-03-11 machineType: m3.medium maxSize: 1 minSize: 1 @@ -70,7 +70,7 @@ metadata: kops.k8s.io/cluster: minimal.example.com name: nodes spec: - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2017-07-28 + image: kope.io/k8s-1.9-debian-jessie-amd64-hvm-ebs-2018-03-11 machineType: t2.medium maxSize: 2 minSize: 2 diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml index c1ab20b716632..73290304b5233 100644 --- a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml @@ -25,7 +25,7 @@ spec: legacy: false kubernetesApiAccess: - 0.0.0.0/0 - kubernetesVersion: v1.4.8 + kubernetesVersion: v1.12.0 masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 networking: @@ -54,7 +54,7 @@ metadata: kops.k8s.io/cluster: minimal.example.com name: master-us-test-1a spec: - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2017-07-28 + image: kope.io/k8s-1.9-debian-jessie-amd64-hvm-ebs-2018-03-11 machineType: m3.medium maxSize: 1 minSize: 1 @@ -74,7 +74,7 @@ metadata: kops.k8s.io/cluster: minimal.example.com name: nodes spec: - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2017-07-28 + image: kope.io/k8s-1.9-debian-jessie-amd64-hvm-ebs-2018-03-11 machineType: t2.medium maxSize: 2 minSize: 2 diff --git a/tests/integration/create_cluster/minimal/options.yaml b/tests/integration/create_cluster/minimal/options.yaml index 211f4faba1210..be32b6109f346 100644 --- a/tests/integration/create_cluster/minimal/options.yaml +++ b/tests/integration/create_cluster/minimal/options.yaml @@ -2,4 +2,4 @@ ClusterName: minimal.example.com Zones: - us-test-1a Cloud: aws -KubernetesVersion: v1.4.8 +KubernetesVersion: v1.12.0 diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 23b3bbeac65a8..71b9b6188131d 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1231,7 +1231,7 @@ func (c *ApplyClusterCmd) BuildNodeUpConfig(assetBuilder *assets.AssetBuilder, i if role == kops.InstanceGroupRoleMaster { for _, etcdCluster := range cluster.Spec.EtcdClusters { - if etcdCluster.Manager != nil { + if etcdCluster.Provider == kops.EtcdProviderTypeManager { p := configBase.Join("manifests/etcd/" + etcdCluster.Name + ".yaml").Path() config.EtcdManifests = append(config.EtcdManifests, p) } diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 24f8249aa950d..1300907a6df8f 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -294,8 +294,8 @@ func (c *populateClusterSpec) run(clientset simple.Clientset) error { { // Note: DefaultOptionsBuilder comes first codeModels = append(codeModels, &components.DefaultsOptionsBuilder{Context: optionsContext}) - codeModels = append(codeModels, &components.EtcdOptionsBuilder{Context: optionsContext}) - codeModels = append(codeModels, &etcdmanager.EtcdManagerOptionsBuilder{Context: optionsContext}) + codeModels = append(codeModels, &components.EtcdOptionsBuilder{OptionsContext: optionsContext}) + codeModels = append(codeModels, &etcdmanager.EtcdManagerOptionsBuilder{OptionsContext: optionsContext}) codeModels = append(codeModels, &nodeauthorizer.OptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.KubeAPIServerOptionsBuilder{OptionsContext: optionsContext}) codeModels = append(codeModels, &components.DockerOptionsBuilder{OptionsContext: optionsContext}) diff --git a/upup/pkg/fi/cloudup/validation_test.go b/upup/pkg/fi/cloudup/validation_test.go index d71497669c0a4..6b93bab726666 100644 --- a/upup/pkg/fi/cloudup/validation_test.go +++ b/upup/pkg/fi/cloudup/validation_test.go @@ -150,7 +150,7 @@ func Test_Validate_No_Classic_With_14(t *testing.T) { Classic: &api.ClassicNetworkingSpec{}, } - expectErrorFromValidate(t, c, "Spec.Networking") + expectErrorFromValidate(t, c, "spec.Networking") } func Test_Validate_Kubenet_With_14(t *testing.T) {