diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 581d58728ce18..679c93fa0722d 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -58,7 +58,10 @@ type CreateClusterOptions struct { AssociatePublicIP bool // Channel is the location of the api.Channel to use for our defaults - Channel string + Channel string + + // The network topology to use + Topology string } func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { @@ -111,6 +114,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.Channel, "channel", api.DefaultChannel, "Channel for default versions and configuration to use") + // Network topology + cmd.Flags().StringVarP(&options.Topology, "topology", "t", "public", "Controls network topology for the cluster. public|private. Default is 'public'.") + return cmd } @@ -359,6 +365,23 @@ func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io } } + + // Network Topology + switch c.Topology{ + case api.TopologyPublic: + cluster.Spec.Topology = &api.TopologySpec{Masters: api.TopologyPublic, Nodes: api.TopologyPublic, BypassBastion: false} + case api.TopologyPrivate: + if c.Networking != "cni" { + return fmt.Errorf("Invalid networking option %s. Currently only '--networking cni' is supported for private topologies", c.Networking) + } + cluster.Spec.Topology = &api.TopologySpec{Masters: api.TopologyPrivate, Nodes: api.TopologyPrivate, BypassBastion: false} + case "": + glog.Warningf("Empty topology. Defaulting to public topology.") + cluster.Spec.Topology = &api.TopologySpec{Masters: api.TopologyPublic, Nodes: api.TopologyPublic, BypassBastion: false} + default: + return fmt.Errorf("Invalid topology %s.", c.Topology) + } + sshPublicKeys := make(map[string][]byte) if c.SSHPublicKey != "" { c.SSHPublicKey = utils.ExpandPath(c.SSHPublicKey) @@ -505,4 +528,4 @@ func parseZoneList(s string) []string { filtered = append(filtered, v) } return filtered -} +} \ No newline at end of file diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go index f0eb6fcf9d271..5ebaa9479e65b 100644 --- a/cmd/kops/update_cluster.go +++ b/cmd/kops/update_cluster.go @@ -30,6 +30,7 @@ import ( "k8s.io/kops/upup/pkg/kutil" "os" "strings" + "k8s.io/kops/pkg/apis/kops" ) type UpdateClusterOptions struct { @@ -188,7 +189,11 @@ func RunUpdateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io fmt.Printf("\n") fmt.Printf("Suggestions:\n") fmt.Printf(" * list nodes: kubectl get nodes --show-labels\n") - fmt.Printf(" * ssh to the master: ssh -i ~/.ssh/id_rsa admin@%s\n", cluster.Spec.MasterPublicName) + if cluster.Spec.Topology.Masters == kops.TopologyPublic { + fmt.Printf(" * ssh to the master: ssh -i ~/.ssh/id_rsa admin@%s\n", cluster.Spec.MasterPublicName) + }else { + fmt.Printf(" * ssh to the bastion: ssh -i ~/.ssh/id_rsa admin@%s\n", cluster.Spec.MasterPublicName) + } fmt.Printf(" * read about installing addons: https://github.com/kubernetes/kops/blob/master/docs/addons.md\n") fmt.Printf("\n") } diff --git a/docs/topology.md b/docs/topology.md new file mode 100644 index 0000000000000..eac362385c956 --- /dev/null +++ b/docs/topology.md @@ -0,0 +1,45 @@ +# Network Topologies in Kops + +Kops supports a number of pre defined network topologies. They are separated into commonly used scenarios, or topologies. + +Each of the supported topologies are listed below, with an example on how to deploy them. + +# AWS + +Kops supports the following topologies on AWS + +| Topology | Value | Description | +| ----------------- |----------- | ----------------------------------------------------------------------------------------------------------- | +| Public Cluster | public | All masters/nodes will be launched in a **public subnet** in the VPC | +| Private Cluster | private | All masters/nodes will be launched in a **private subnet** in the VPC | + + +[More information](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html) on Public and Private subnets in AWS + +Notes on subnets + +##### Public Subnet +If a subnet's traffic is routed to an Internet gateway, the subnet is known as a public subnet. + +##### Private Subnet +If a subnet doesn't have a route to the Internet gateway, the subnet is known as a private subnet. + +Private topologies *will* have public access via the Kubernetes API and an (optional) SSH bastion instance. + +# Defining a topology on create + +To specify a topology use the `--topology` or `-t` flag as in : + +``` +kops create cluster ... --topology public|private +``` + +# Troubleshooting + +- Right now we require `-networking cni` for all private topologies. +- Right now a manual install of weave is required for private topologies. +- Right now upgrading from a public cluster to a private cluster is considered very **experimental** + +``` +kubectl create -f https://git.io/weave-kube +``` \ No newline at end of file diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 5eb4394f1124a..769e6d91c9e26 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -29,7 +29,7 @@ import ( type Cluster struct { unversioned.TypeMeta `json:",inline"` - ObjectMeta `json:"metadata,omitempty"` + ObjectMeta `json:"metadata,omitempty"` Spec ClusterSpec `json:"spec,omitempty"` } @@ -77,6 +77,11 @@ type ClusterSpec struct { // NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC) NetworkID string `json:"networkID,omitempty"` + // Topology defines the type of network topology to use on the cluster - default public + // This is heavily weighted towards AWS for the time being, but should also be agnostic enough + // to port out to GCE later if needed + Topology *TopologySpec `json:"topology,omitempty"` + // SecretStore is the VFS path to where secrets are stored SecretStore string `json:"secretStore,omitempty"` // KeyStore is the VFS path to where SSL keys and certificates are stored @@ -275,7 +280,14 @@ type EtcdMemberSpec struct { type ClusterZoneSpec struct { Name string `json:"name,omitempty"` - CIDR string `json:"cidr,omitempty"` + + // For Private network topologies we need to have 2 + // CIDR blocks. + // 1 - Utility (Public) Subnets + // 2 - Operating (Private) Subnets + + PrivateCIDR string `json:"privateCIDR,omitempty"` + CIDR string `json:"cidr,omitempty"` // ProviderID is the cloud provider id for the objects associated with the zone (the subnet on AWS) ProviderID string `json:"id,omitempty"` @@ -341,10 +353,10 @@ func (c *Cluster) FillDefaults() error { // OK } else if c.Spec.Networking.Kubenet != nil { // OK - } else if c.Spec.Networking.External != nil { - // OK } else if c.Spec.Networking.CNI != nil { // OK + } else if c.Spec.Networking.External != nil { + // OK } else { // No networking model selected; choose Kubenet c.Spec.Networking.Kubenet = &KubenetNetworkingSpec{} @@ -414,21 +426,27 @@ func FindLatestKubernetesVersion() (string, error) { func (z *ClusterZoneSpec) performAssignments(c *Cluster) error { if z.CIDR == "" { - cidr, err := z.assignCIDR(c) + err := z.assignCIDR(c) if err != nil { return err } - glog.Infof("Assigned CIDR %s to zone %s", cidr, z.Name) - z.CIDR = cidr } - return nil } -func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) { +// Will generate a CIDR block based on the last character in +// the cluster.Spec.Zones structure. +// +func (z *ClusterZoneSpec) assignCIDR(c *Cluster) error { // TODO: We probably could query for the existing subnets & allocate appropriately // for now we'll require users to set CIDRs themselves + // Used in calculating private subnet blocks (if needed only) + needsPrivateBlock := false + if c.Spec.Topology.Masters == TopologyPrivate || c.Spec.Topology.Nodes == TopologyPrivate { + needsPrivateBlock = true + } + lastCharMap := make(map[byte]bool) for _, nodeZone := range c.Spec.Zones { lastChar := nodeZone.Name[len(nodeZone.Name)-1] @@ -452,13 +470,13 @@ func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) { } } if index == -1 { - return "", fmt.Errorf("zone not configured: %q", z.Name) + return fmt.Errorf("zone not configured: %q", z.Name) } } _, cidr, err := net.ParseCIDR(c.Spec.NetworkCIDR) if err != nil { - return "", fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR) + return fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR) } networkLength, _ := cidr.Mask.Size() @@ -475,14 +493,49 @@ func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) { subnetIP := make(net.IP, len(ip4)) binary.BigEndian.PutUint32(subnetIP, n) subnetCIDR := subnetIP.String() + "/" + strconv.Itoa(networkLength) + z.CIDR = subnetCIDR glog.V(2).Infof("Computed CIDR for subnet in zone %q as %q", z.Name, subnetCIDR) - return subnetCIDR, nil + glog.Infof("Assigned CIDR %s to zone %s", subnetCIDR, z.Name) + + if needsPrivateBlock { + m := binary.BigEndian.Uint32(ip4) + // All Private CIDR blocks are at the end of our range + m += uint32(index+len(c.Spec.Zones)) << uint(32-networkLength) + privSubnetIp := make(net.IP, len(ip4)) + binary.BigEndian.PutUint32(privSubnetIp, m) + privCIDR := privSubnetIp.String() + "/" + strconv.Itoa(networkLength) + z.PrivateCIDR = privCIDR + glog.V(2).Infof("Computed Private CIDR for subnet in zone %q as %q", z.Name, privCIDR) + glog.Infof("Assigned Private CIDR %s to zone %s", privCIDR, z.Name) + } + + return nil } - return "", fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR) + return fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR) } // SharedVPC is a simple helper function which makes the templates for a shared VPC clearer func (c *Cluster) SharedVPC() bool { return c.Spec.NetworkID != "" } + +// -------------------------------------------------------------------------------------------- +// Network Topology functions for template parsing +// +// Each of these functions can be used in the model templates +// The go template package currently only supports boolean +// operations, so the logic is mapped here as *Cluster functions. +// +// A function will need to be defined for all new topologies, if we plan to use them in the +// model templates. +// -------------------------------------------------------------------------------------------- +func (c *Cluster) IsTopologyPrivate() bool { + return (c.Spec.Topology.Masters == TopologyPrivate && c.Spec.Topology.Nodes == TopologyPrivate) +} +func (c *Cluster) IsTopologyPublic() bool { + return (c.Spec.Topology.Masters == TopologyPublic && c.Spec.Topology.Nodes == TopologyPublic) +} +func (c *Cluster) IsTopologyPrivateMasters() bool { + return (c.Spec.Topology.Masters == TopologyPrivate && c.Spec.Topology.Nodes == TopologyPublic) +} diff --git a/pkg/apis/kops/topology.go b/pkg/apis/kops/topology.go new file mode 100644 index 0000000000000..2381bd3843e81 --- /dev/null +++ b/pkg/apis/kops/topology.go @@ -0,0 +1,35 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kops + +const ( + TopologyPublic = "public" + TopologyPrivate = "private" +) + +type TopologySpec struct { + // The environment to launch the Kubernetes masters in public|private + Masters string `json:"masters,omitempty"` + + // The environment to launch the Kubernetes nodes in public|private + Nodes string `json:"nodes,omitempty"` + + // Controls if a private topology should deploy a bastion host or not + // The bastion host is designed to be a simple, and secure bridge between + // the public subnet and the private subnet + BypassBastion bool `json:"bypassBastion,omitempty"` +} \ No newline at end of file diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index b84ad233139d2..c34c291d3e479 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -78,6 +78,11 @@ type ClusterSpec struct { // NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC) NetworkID string `json:"networkID,omitempty"` + // Topology defines the type of network topology to use on the cluster - default public + // This is heavily weighted towards AWS for the time being, but should also be agnostic enough + // to port out to GCE later if needed + Topology *TopologySpec `json:"topology,omitempty"` + // SecretStore is the VFS path to where secrets are stored SecretStore string `json:"secretStore,omitempty"` // KeyStore is the VFS path to where SSL keys and certificates are stored diff --git a/pkg/apis/kops/v1alpha1/topology.go b/pkg/apis/kops/v1alpha1/topology.go new file mode 100644 index 0000000000000..6f5e6a77127e1 --- /dev/null +++ b/pkg/apis/kops/v1alpha1/topology.go @@ -0,0 +1,35 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +const ( + TopologyPublic = "public" + TopologyPrivate = "private" +) + +type TopologySpec struct { + // The environment to launch the Kubernetes masters in public|private + Masters string `json:"masters,omitempty"` + + // The environment to launch the Kubernetes nodes in public|private + Nodes string `json:"nodes,omitempty"` + + // Controls if a private topology should deploy a bastion host or not + // The bastion host is designed to be a simple, and secure bridge between + // the public subnet and the private subnet + BypassBastion bool `json:"bypassBastion,omitempty"` +} \ No newline at end of file diff --git a/pkg/apis/kops/validation.go b/pkg/apis/kops/validation.go index 3224d3fd17257..be62bbb49ae4c 100644 --- a/pkg/apis/kops/validation.go +++ b/pkg/apis/kops/validation.go @@ -292,6 +292,21 @@ func (c *Cluster) Validate(strict bool) error { } } + // Topology support + if c.Spec.Topology.Masters != "" && c.Spec.Topology.Nodes != "" { + if c.Spec.Topology.Masters != TopologyPublic && c.Spec.Topology.Masters != TopologyPrivate { + return fmt.Errorf("Invalid Masters value for Topology") + } else if c.Spec.Topology.Nodes != TopologyPublic && c.Spec.Topology.Nodes != TopologyPrivate { + return fmt.Errorf("Invalid Nodes value for Topology") + // Until we support other topologies - these must match + } else if c.Spec.Topology.Masters != c.Spec.Topology.Nodes { + return fmt.Errorf("Topology Nodes must match Topology Masters") + } + + }else{ + return fmt.Errorf("Topology requires non-nil values for Masters and Nodes") + } + // Etcd { if len(c.Spec.EtcdClusters) == 0 { diff --git a/upup/models/cloudup/_aws/master/_master_asg/master_asg.yaml b/upup/models/cloudup/_aws/master/_master_asg/master_asg.yaml index c75c15a0321ff..790b40aa7a878 100644 --- a/upup/models/cloudup/_aws/master/_master_asg/master_asg.yaml +++ b/upup/models/cloudup/_aws/master/_master_asg/master_asg.yaml @@ -8,7 +8,12 @@ launchConfiguration/{{ $m.Name }}.masters.{{ ClusterName }}: iamInstanceProfile: iamInstanceProfile/masters.{{ ClusterName }} imageId: {{ $m.Spec.Image }} instanceType: {{ $m.Spec.MachineType }} + {{ if IsTopologyPublic }} associatePublicIP: {{ WithDefaultBool $m.Spec.AssociatePublicIP true }} + {{ end }} + {{ if IsTopologyPrivate }} + associatePublicIP: false + {{ end }} userData: resources/nodeup.sh {{ $m.Name }} rootVolumeSize: {{ or $m.Spec.RootVolumeSize "20" }} rootVolumeType: {{ or $m.Spec.RootVolumeType "gp2" }} @@ -21,7 +26,13 @@ autoscalingGroup/{{ $m.Name }}.masters.{{ ClusterName }}: maxSize: {{ $m.Spec.MaxSize }} subnets: {{ range $z := $m.Spec.Zones }} + {{ if IsTopologyPublic }} - subnet/{{ $z }}.{{ ClusterName }} + {{ end }} + {{ if IsTopologyPrivate }} + - subnet/private-{{ $z }}.{{ ClusterName }} + {{ end }} + {{ end }} launchConfiguration: launchConfiguration/{{ $m.Name }}.masters.{{ ClusterName }} tags: diff --git a/upup/models/cloudup/_aws/master/_not_master_lb/not_master_lb.yaml b/upup/models/cloudup/_aws/master/_not_master_lb/not_master_lb.yaml index c16aab519af6d..c9e421b93855c 100644 --- a/upup/models/cloudup/_aws/master/_not_master_lb/not_master_lb.yaml +++ b/upup/models/cloudup/_aws/master/_not_master_lb/not_master_lb.yaml @@ -10,4 +10,4 @@ securityGroupRule/https-external-to-master-{{ $index }}: protocol: tcp fromPort: 443 toPort: 443 -{{ end }} +{{ end }} \ No newline at end of file diff --git a/upup/models/cloudup/_aws/master/master.yaml b/upup/models/cloudup/_aws/master/master.yaml index bb5c6628af544..501fe01caa05b 100644 --- a/upup/models/cloudup/_aws/master/master.yaml +++ b/upup/models/cloudup/_aws/master/master.yaml @@ -28,6 +28,7 @@ securityGroupRule/master-egress: cidr: 0.0.0.0/0 # SSH is open to AdminCIDR set +{{ if IsTopologyPublic }} {{ range $index, $cidr := AdminCIDR }} securityGroupRule/ssh-external-to-master-{{ $index }}: securityGroup: securityGroup/masters.{{ ClusterName }} @@ -36,6 +37,7 @@ securityGroupRule/ssh-external-to-master-{{ $index }}: fromPort: 22 toPort: 22 {{ end }} +{{ end }} # Masters can talk to masters securityGroupRule/all-master-to-master: @@ -46,3 +48,9 @@ securityGroupRule/all-master-to-master: securityGroupRule/all-master-to-node: securityGroup: securityGroup/nodes.{{ ClusterName }} sourceGroup: securityGroup/masters.{{ ClusterName }} + +{{ if and WithBastion IsTopologyPrivate }} +securityGroupRule/bastion-to-master: + securityGroup: securityGroup/masters.{{ ClusterName }} + sourceGroup: securityGroup/bastion.{{ ClusterName }} +{{ end }} \ No newline at end of file diff --git a/upup/models/cloudup/_aws/topologies/_topology_private/bastion.yaml b/upup/models/cloudup/_aws/topologies/_topology_private/bastion.yaml new file mode 100644 index 0000000000000..8d2f05528f129 --- /dev/null +++ b/upup/models/cloudup/_aws/topologies/_topology_private/bastion.yaml @@ -0,0 +1,142 @@ +# --------------------------------------------------------------- +# +# Private Network Topology in AWS +# +# Inspired by https://github.com/kubernetes/kops/issues/428 +# +# --------------------------------------------------------------- + +{{ if WithBastion }} + +# --------------------------------------------------------------- +# Security Group - Bastion +# +# The security group that the bastion lives in +# --------------------------------------------------------------- +securityGroup/bastion.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + description: 'Security group for bastion' + removeExtraRules: + - port=22 +securityGroupRule/bastion-egress: + securityGroup: securityGroup/nodes.{{ ClusterName }} + egress: true + cidr: 0.0.0.0/0 +# TODO Kris - I don't think we need to open these +#securityGroupRule/all-node-to-bastion: +# securityGroup: securityGroup/bastion.{{ ClusterName }} +# sourceGroup: securityGroup/nodes.{{ ClusterName }} +#securityGroupRule/all-master-to-bastion: +# securityGroup: securityGroup/bastion.{{ ClusterName }} +# sourceGroup: securityGroup/masters.{{ ClusterName }} +securityGroupRule/ssh-external-to-bastion: + securityGroup: securityGroup/bastion.{{ ClusterName }} + sourceGroup: securityGroup/bastion-elb.{{ ClusterName }} + protocol: tcp + fromPort: 22 + toPort: 22 + +# --------------------------------------------------------------- +# Security Group - Bastion->Nodes +# +# If we are creating a bastion, we need to poke a hole in the +# firewall for it to talk to our masters +# --------------------------------------------------------------- +securityGroupRule/all-bastion-to-master: + securityGroup: securityGroup/nodes.{{ ClusterName }} + sourceGroup: securityGroup/bastion.{{ ClusterName }} + +# --------------------------------------------------------------- +# Security Group - Bastion ELB +# +# The security group that the bastion lives in +# --------------------------------------------------------------- +securityGroup/bastion-elb.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + description: 'Security group for bastion ELB' + removeExtraRules: + - port=22 +securityGroupRule/bastion-elb-egress: + securityGroup: securityGroup/bastion-elb.{{ ClusterName }} + egress: true + cidr: 0.0.0.0/0 +securityGroupRule/ssh-external-to-bastion-elb: + securityGroup: securityGroup/bastion-elb.{{ ClusterName }} + cidr: 0.0.0.0/0 + protocol: tcp + fromPort: 22 + toPort: 22 + + +# --------------------------------------------------------------- +# Bastion Load Balancer +# +# This is the ELB in front of the privately hosted bastion ASG +# --------------------------------------------------------------- +loadBalancer/bastion.{{ ClusterName }}: + id: {{ GetELBName32 "bastion" }} + securityGroups: + - securityGroup/bastion-elb.{{ ClusterName }} + subnets: + {{ range $zone := .Zones }} + - subnet/utility-{{ $zone.Name }}.{{ ClusterName }} + {{ end }} + listeners: + 22: { instancePort: 22 } +loadBalancerAttachment/bastion-elb-attachment.{{ ClusterName }}: + loadBalancer: loadBalancer/bastion.{{ ClusterName }} + autoscalingGroup: autoscalingGroup/bastion.{{ ClusterName }} + + +# --------------------------------------------------------------- +# ASG - The Bastion itself +# +# Define the bastion host. Hard coding to a t2.small for now. +# we probably want to abstract this out in a later feature. +# +# The bastion host will live in one of the utility subnets +# created in the private topology. The bastion host will have +# port 22 TCP open to 0.0.0.0/0. And will have internal SSH +# access to all private subnets. +# +# --------------------------------------------------------------- +launchConfiguration/bastion.{{ ClusterName }}: + sshKey: sshKey/{{ SSHKeyName }} + securityGroups: + - securityGroup/bastion.{{ ClusterName }} + iamInstanceProfile: iamInstanceProfile/masters.{{ ClusterName }} + imageId: {{ GetBastionImageId }} + instanceType: t2.small + associatePublicIP: false + rootVolumeSize: 20 + rootVolumeType: gp2 +autoscalingGroup/bastion.{{ ClusterName }}: + minSize: 1 + maxSize: 1 + subnets: + - subnet/private-{{ GetBastionZone }}.{{ ClusterName }} + launchConfiguration: launchConfiguration/bastion.{{ ClusterName }} + tags: + Name: bastion-{{ GetBastionZone }}.{{ ClusterName }} + KubernetesCluster: {{ ClusterName }} + +# --------------------------------------------------------------- +# TODO Kris +# +# So as it stands we are *NOT* defining a friendly CNAME for the +# bastion ELB. I think this is a good thing. +# +# If we came up with a formula EG: bastion. we could +# be exposing ourselves to a few threats :) +# +# I think it's best in this situation to err on the side of +# caution and force the end user to define something convenient +# on their own. +# +# TLDR; If you want a friendly CNAME for your bastion - you have +# to build it yourself. Kops won't support that +# +# Right? +# --------------------------------------------------------------- + +{{ end }} diff --git a/upup/models/cloudup/_aws/topologies/_topology_private/network.yaml b/upup/models/cloudup/_aws/topologies/_topology_private/network.yaml new file mode 100644 index 0000000000000..c1fe962406f84 --- /dev/null +++ b/upup/models/cloudup/_aws/topologies/_topology_private/network.yaml @@ -0,0 +1,258 @@ +# --------------------------------------------------------------- +# +# Private Network Topology in AWS +# +# Inspired by https://github.com/kubernetes/kops/issues/428 +# +# --------------------------------------------------------------- + + +# --------------------------------------------------------------- +# VPC +# +# This is a single VPC that will hold all networking componets for +# a k8s cluster +# +# --------------------------------------------------------------- +vpc/{{ ClusterName }}: + id: {{ .NetworkID }} + shared: {{ SharedVPC }} + cidr: {{ .NetworkCIDR }} + enableDnsSupport: true + enableDnsHostnames: true + +# --------------------------------------------------------------- +# DHCP +# +# If this is not a shared VPC +# (There is more than one availability zone for this cluster) +# +# Also add support for us-east-1 +# --------------------------------------------------------------- +{{ if not SharedVPC }} +dhcpOptions/{{ ClusterName }}: + domainNameServers: AmazonProvidedDNS +{{ if eq Region "us-east-1" }} + domainName: ec2.internal +{{ else }} + domainName: {{ Region }}.compute.internal +{{ end }} +vpcDHDCPOptionsAssociation/{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + dhcpOptions: dhcpOptions/{{ ClusterName }} +{{ end }} + +# --------------------------------------------------------------- +# Internet Gateway +# +# This is the main entry point to the cluster. There will be a +# route table associated with the gateway. +# --------------------------------------------------------------- +internetGateway/{{ ClusterName }}: + shared: {{ SharedVPC }} + vpc: vpc/{{ ClusterName }} + +# --------------------------------------------------------------- +# Main Route Table +# +# The main route table associated with the Internet Gateway +# --------------------------------------------------------------- +routeTable/main-{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + +# --------------------------------------------------------------- +# Main Routes +# +# Routes for the Main Route Table +# --------------------------------------------------------------- +route/wan: + routeTable: routeTable/main-{{ ClusterName }} + cidr: 0.0.0.0/0 + internetGateway: internetGateway/{{ ClusterName }} + vpc: vpc/{{ ClusterName }} + +# --------------------------------------------------------------- +# Zones (Availability Zones) +# +# For every availability zone +# - 1 Utility/Public subnet +# - 1 NGW for the private subnet to NAT to +# - 1 Route Table Association to the Main Route Table +# - 1 Private subnet (to hold the instances) +# --------------------------------------------------------------- +{{ range $zone := .Zones }} + +# --------------------------------------------------------------- +# Utility Subnet +# +# This is the public subnet that will hold the route to the +# gateway, the NAT gateway +# --------------------------------------------------------------- +subnet/utility-{{ $zone.Name }}.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + availabilityZone: {{ $zone.Name }} + cidr: {{ $zone.CIDR }} + id: {{ $zone.ProviderID }} + shared: {{ SharedZone $zone }} + + +{{ if not (SharedZone $zone) }} + +# --------------------------------------------------------------- +# Utility Subnet Route Table Associations +# +# Map the Utility subnet to the Main route table +# --------------------------------------------------------------- +routeTableAssociation/main-{{ $zone.Name }}.{{ ClusterName }}: + routeTable: routeTable/main-{{ ClusterName }} + subnet: subnet/utility-{{ $zone.Name }}.{{ ClusterName }} + +# --------------------------------------------------------------- +# Elastic IP +# +# Every NGW needs a public (Elastic) IP address, every private +# subnet needs a NGW, lets create it. We tie it to a subnet +# so we can track it in AWS +# --------------------------------------------------------------- +elasticIP/{{ $zone.Name }}.{{ ClusterName }}: + subnet: subnet/utility-{{ $zone.Name }}.{{ ClusterName }} + +# --------------------------------------------------------------- +# NAT Gateway +# +# All private subnets will need a NGW +# +# The instances in the private subnet can access the Internet by +# using a network address translation (NAT) gateway that resides +# in the public subnet. +# --------------------------------------------------------------- +ngw/{{ $zone.Name }}.{{ ClusterName }}: + elasticIp: elasticIP/{{ $zone.Name }}.{{ ClusterName }} + subnet: subnet/utility-{{ $zone.Name }}.{{ ClusterName }} + +# --------------------------------------------------------------- +# Private Subnet +# +# This is the private subnet for each AZ +# --------------------------------------------------------------- +subnet/private-{{ $zone.Name }}.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + availabilityZone: {{ $zone.Name }} + cidr: {{ $zone.PrivateCIDR }} + id: {{ $zone.ProviderID }} + shared: {{ SharedZone $zone }} + +# --------------------------------------------------------------- +# Private Route Table +# +# The private route table that will route to the NAT Gateway +# --------------------------------------------------------------- +routeTable/private-{{ $zone.Name }}.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + +# --------------------------------------------------------------- +# Private Subnet Route Table Associations +# +# Map the Private subnet to the Private route table +# --------------------------------------------------------------- +routeTableAssociation/private-{{ $zone.Name }}.{{ ClusterName }}: + routeTable: routeTable/private-{{ $zone.Name }}.{{ ClusterName }} + subnet: subnet/private-{{ $zone.Name }}.{{ ClusterName }} + +# --------------------------------------------------------------- +# Private Routes +# +# Routes for the private route table. +# Will route to the NAT Gateway +# --------------------------------------------------------------- +route/private-{{ $zone.Name }}.{{ ClusterName }}: + routeTable: routeTable/private-{{ $zone.Name }}.{{ ClusterName }} + cidr: 0.0.0.0/0 + vpc: vpc/{{ ClusterName }} + natGateway: ngw/{{ $zone.Name }}.{{ ClusterName }} + + +{{ end }} # SharedVPC +{{ end }} # For Each Zone + +# --------------------------------------------------------------- +# Load Balancer - API +# +# This is the load balancer in front of the Kubernetes API +# --------------------------------------------------------------- +loadBalancer/api.{{ ClusterName }}: + id: {{ GetELBName32 "api" }} + securityGroups: + - securityGroup/api-elb.{{ ClusterName }} + subnets: + {{ range $zone := .Zones }} + - subnet/utility-{{ $zone.Name }}.{{ ClusterName }} + {{ end }} + listeners: + 443: { instancePort: 443 } + +# --------------------------------------------------------------- +# Kube-Proxy - Healthz - 10249 +# +# HealthCheck for the kubernetes API via the kube-proxy +# --------------------------------------------------------------- +loadBalancerHealthChecks/api.{{ ClusterName }}: + loadBalancer: loadBalancer/api.{{ ClusterName }} + target: TCP:443 + healthyThreshold: 2 + unhealthyThreshold: 2 + interval: 10 + timeout: 5 +securityGroupRule/kube-proxy-api-elb: + securityGroup: securityGroup/masters.{{ ClusterName }} + sourceGroup: securityGroup/api-elb.{{ ClusterName }} + protocol: tcp + fromPort: 443 + toPort: 443 + + +# --------------------------------------------------------------- +# Load Balancer - Masters +# +# Attach each master ASG to the ELB +# --------------------------------------------------------------- +{{ range $m := Masters }} +loadBalancerAttachment/api-elb-attachment.{{ $m.Name }}.{{ ClusterName }}: + loadBalancer: loadBalancer/api.{{ ClusterName }} + autoscalingGroup: autoscalingGroup/{{ $m.Name }}.masters.{{ ClusterName }} +{{ end }} + + +# --------------------------------------------------------------- +# Security Group - API ELB +# +# This is the security group that is external facing. These are +# the public rules for kubernetes! +# --------------------------------------------------------------- +securityGroup/api-elb.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + description: 'Security group for api ELB' + removeExtraRules: + - port=22 +securityGroupRule/api-elb-egress: + securityGroup: securityGroup/api-elb.{{ ClusterName }} + egress: true + cidr: 0.0.0.0/0 +securityGroupRule/https-api-elb: + securityGroup: securityGroup/api-elb.{{ ClusterName }} + cidr: 0.0.0.0/0 + protocol: tcp + fromPort: 443 + toPort: 443 + +# --------------------------------------------------------------- +# DNS - Api +# +# This will point our DNS to the load balancer, and put the pieces +# together for kubectl to be work +# --------------------------------------------------------------- +dnsZone/{{ .DNSZone }}: {} +dnsName/{{ .MasterPublicName }}: + Zone: dnsZone/{{ .DNSZone }} + ResourceType: "A" + TargetLoadBalancer: loadBalancer/api.{{ ClusterName }} \ No newline at end of file diff --git a/upup/models/cloudup/_aws/topologies/_topology_private/nodes.yaml b/upup/models/cloudup/_aws/topologies/_topology_private/nodes.yaml new file mode 100644 index 0000000000000..06787500fda9b --- /dev/null +++ b/upup/models/cloudup/_aws/topologies/_topology_private/nodes.yaml @@ -0,0 +1,79 @@ +# --------------------------------------------------------------- +# +# Private Network Topology in AWS +# +# Inspired by https://github.com/kubernetes/kops/issues/428 +# +# --------------------------------------------------------------- + + +# --------------------------------------------------------------- +# IAM - Nodes +# --------------------------------------------------------------- +iamRole/nodes.{{ ClusterName }}: + rolePolicyDocument: resources/iam/kubernetes-node-role.json +iamRolePolicy/nodes.{{ ClusterName }}: + role: iamRole/nodes.{{ ClusterName }} + policyDocument: resources/iam/kubernetes-node-policy.json +iamInstanceProfile/nodes.{{ ClusterName }}: {} +iamInstanceProfileRole/nodes.{{ ClusterName }}: + instanceProfile: iamInstanceProfile/nodes.{{ ClusterName }} + role: iamRole/nodes.{{ ClusterName }} + +# --------------------------------------------------------------- +# Security Group - Nodes +# --------------------------------------------------------------- +securityGroup/nodes.{{ ClusterName }}: + vpc: vpc/{{ ClusterName }} + description: 'Security group for nodes' + removeExtraRules: + - port=22 +securityGroupRule/node-egress: + securityGroup: securityGroup/nodes.{{ ClusterName }} + egress: true + cidr: 0.0.0.0/0 +securityGroupRule/all-node-to-node: + securityGroup: securityGroup/nodes.{{ ClusterName }} + sourceGroup: securityGroup/nodes.{{ ClusterName }} +securityGroupRule/all-node-to-master: + securityGroup: securityGroup/masters.{{ ClusterName }} + sourceGroup: securityGroup/nodes.{{ ClusterName }} + + +{{ range $ig := NodeSets }} +# --------------------------------------------------------------- +# AutoScaleGroup - Nodes +# +# The AutoScaleGroup for the Nodes +# --------------------------------------------------------------- +launchConfiguration/{{ $ig.Name }}.{{ ClusterName }}: + sshKey: sshKey/{{ SSHKeyName }} + securityGroups: + - securityGroup/nodes.{{ ClusterName }} + iamInstanceProfile: iamInstanceProfile/nodes.{{ ClusterName }} + imageId: {{ $ig.Spec.Image }} + instanceType: {{ $ig.Spec.MachineType }} + associatePublicIP: false + userData: resources/nodeup.sh {{ $ig.Name }} + rootVolumeSize: {{ or $ig.Spec.RootVolumeSize "20" }} + rootVolumeType: {{ or $ig.Spec.RootVolumeType "gp2" }} +{{ if $ig.Spec.MaxPrice }} + spotPrice: "{{ $ig.Spec.MaxPrice }}" +{{ end }} +autoscalingGroup/{{ $ig.Name }}.{{ ClusterName }}: + launchConfiguration: launchConfiguration/{{ $ig.Name }}.{{ ClusterName }} + minSize: {{ or $ig.Spec.MinSize 2 }} + maxSize: {{ or $ig.Spec.MaxSize 2 }} + subnets: +{{ range $zone := $ig.Spec.Zones }} + # Nodes in private topology get launched into the private subnet + - subnet/private-{{ $zone }}.{{ ClusterName }} +{{ end }} + tags: + {{ range $k, $v := CloudTags $ig }} + {{ $k }}: "{{ $v }}" + {{ end }} + + + +{{ end }} diff --git a/upup/models/cloudup/_aws/network.yaml b/upup/models/cloudup/_aws/topologies/_topology_public/network.yaml similarity index 95% rename from upup/models/cloudup/_aws/network.yaml rename to upup/models/cloudup/_aws/topologies/_topology_public/network.yaml index ee14fdfd11735..325e3ec6e0ada 100644 --- a/upup/models/cloudup/_aws/network.yaml +++ b/upup/models/cloudup/_aws/topologies/_topology_public/network.yaml @@ -5,6 +5,7 @@ vpc/{{ ClusterName }}: enableDnsSupport: true enableDnsHostnames: true + {{ if not SharedVPC }} # TODO: would be good to create these as shared, to verify them dhcpOptions/{{ ClusterName }}: @@ -24,7 +25,6 @@ internetGateway/{{ ClusterName }}: shared: {{ SharedVPC }} vpc: vpc/{{ ClusterName }} -# Name must match name in convert_kubeup_cluster.go routeTable/{{ ClusterName }}: vpc: vpc/{{ ClusterName }} @@ -35,7 +35,6 @@ route/0.0.0.0/0: vpc: vpc/{{ ClusterName }} {{ range $zone := .Zones }} - subnet/{{ $zone.Name }}.{{ ClusterName }}: vpc: vpc/{{ ClusterName }} availabilityZone: {{ $zone.Name }} @@ -47,6 +46,5 @@ subnet/{{ $zone.Name }}.{{ ClusterName }}: routeTableAssociation/{{ $zone.Name }}.{{ ClusterName }}: routeTable: routeTable/{{ ClusterName }} subnet: subnet/{{ $zone.Name }}.{{ ClusterName }} -{{ end}} - {{ end }} +{{ end }} \ No newline at end of file diff --git a/upup/models/cloudup/_aws/nodes.yaml b/upup/models/cloudup/_aws/topologies/_topology_public/nodes.yaml similarity index 100% rename from upup/models/cloudup/_aws/nodes.yaml rename to upup/models/cloudup/_aws/topologies/_topology_public/nodes.yaml diff --git a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/etc/kubernetes/manifests/kube-apiserver.manifest.template b/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/etc/kubernetes/manifests/kube-apiserver.manifest.template index 926687379c417..3e9684ea46b9a 100644 --- a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/etc/kubernetes/manifests/kube-apiserver.manifest.template +++ b/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/etc/kubernetes/manifests/kube-apiserver.manifest.template @@ -2,7 +2,9 @@ apiVersion: v1 kind: Pod metadata: annotations: +{{ if IsTopologyPublic }} dns.alpha.kubernetes.io/external: {{ .MasterPublicName }} +{{ end }} dns.alpha.kubernetes.io/internal: {{ .MasterInternalName }} name: kube-apiserver namespace: kube-system diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 9c913086b0eee..b15e13cf17417 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -79,7 +79,9 @@ type ApplyClusterCmd struct { DryRun bool } + func (c *ApplyClusterCmd) Run() error { + if c.InstanceGroups == nil { list, err := c.Clientset.InstanceGroups(c.Cluster.Name).List(k8sapi.ListOptions{}) if err != nil { @@ -293,6 +295,7 @@ func (c *ApplyClusterCmd) Run() error { "securityGroupRule": &awstasks.SecurityGroupRule{}, "subnet": &awstasks.Subnet{}, "vpc": &awstasks.VPC{}, + "ngw": &awstasks.NatGateway{}, "vpcDHDCPOptionsAssociation": &awstasks.VPCDHCPOptionsAssociation{}, // ELB @@ -553,8 +556,7 @@ func (c *ApplyClusterCmd) Run() error { if err != nil { return fmt.Errorf("error running tasks: %v", err) } - - err = target.Finish(taskMap) + err = target.Finish(taskMap) //This will finish the apply, and print the changes if err != nil { return fmt.Errorf("error closing target: %v", err) } @@ -676,4 +678,4 @@ func ChannelForCluster(c *api.Cluster) (*api.Channel, error) { channelLocation = api.DefaultChannel } return api.LoadChannel(channelLocation) -} +} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/awstasks/elastic_ip.go b/upup/pkg/fi/cloudup/awstasks/elastic_ip.go index c39175778df6f..812fde1b618bc 100644 --- a/upup/pkg/fi/cloudup/awstasks/elastic_ip.go +++ b/upup/pkg/fi/cloudup/awstasks/elastic_ip.go @@ -17,26 +17,31 @@ limitations under the License. package awstasks import ( - "fmt" - + //"fmt" + // "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/glog" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "fmt" ) //go:generate fitask -type=ElasticIP + +// Elastic IP +// Representation the EIP AWS task type ElasticIP struct { - Name *string + Name *string + ID *string + PublicIP *string + - ID *string - PublicIP *string - // Because ElasticIPs don't supporting tagging (sadly), we instead tag on - // a different resource - TagUsingKey *string - TagOnResource fi.Task + // Allow support for associated subnets + // If you need another resource to tag on (ebs volume) + // you must add it + Subnet *Subnet } var _ fi.HasAddress = &ElasticIP{} @@ -52,41 +57,23 @@ func (e *ElasticIP) FindAddress(context *fi.Context) (*string, error) { return actual.PublicIP, nil } +// +// Find (public wrapper for find() +// func (e *ElasticIP) Find(context *fi.Context) (*ElasticIP, error) { return e.find(context.Cloud.(awsup.AWSCloud)) } -func (e *ElasticIP) findTagOnResourceID(cloud awsup.AWSCloud) (*string, error) { - if e.TagOnResource == nil { - return nil, nil - } - - var tagOnResource TaggableResource - var ok bool - if tagOnResource, ok = e.TagOnResource.(TaggableResource); !ok { - return nil, fmt.Errorf("TagOnResource must implement TaggableResource (type is %T)", e.TagOnResource) - } - - id, err := tagOnResource.FindResourceID(cloud) - if err != nil { - return nil, fmt.Errorf("error trying to find id of TagOnResource: %v", err) - } - return id, err -} - +// Will attempt to look up the elastic IP from AWS func (e *ElasticIP) find(cloud awsup.AWSCloud) (*ElasticIP, error) { publicIP := e.PublicIP allocationID := e.ID - tagOnResourceID, err := e.findTagOnResourceID(cloud) - if err != nil { - return nil, err - } // Find via tag on foreign resource - if allocationID == nil && publicIP == nil && e.TagUsingKey != nil && tagOnResourceID != nil { + if allocationID == nil && publicIP == nil && e.Subnet.ID != nil { var filters []*ec2.Filter - filters = append(filters, awsup.NewEC2Filter("key", *e.TagUsingKey)) - filters = append(filters, awsup.NewEC2Filter("resource-id", *tagOnResourceID)) + filters = append(filters, awsup.NewEC2Filter("key", "AssociatedElasticIp")) + filters = append(filters, awsup.NewEC2Filter("resource-id", *e.Subnet.ID)) request := &ec2.DescribeTagsInput{ Filters: filters, @@ -135,38 +122,55 @@ func (e *ElasticIP) find(cloud awsup.AWSCloud) (*ElasticIP, error) { PublicIP: a.PublicIp, } - // These two are weird properties; we copy them so they don't come up as changes - actual.TagUsingKey = e.TagUsingKey - actual.TagOnResource = e.TagOnResource + actual.Subnet = e.Subnet e.ID = actual.ID return actual, nil } - return nil, nil } +// The Run() function is called to execute this task. +// This is the main entry point of the task, and will actually +// connect our internal resource representation to an actual +// resource in AWS func (e *ElasticIP) Run(c *fi.Context) error { return fi.DefaultDeltaRunMethod(e, c) } +// Validation for the resource. EIPs are simple, so virtually no +// validation func (s *ElasticIP) CheckChanges(a, e, changes *ElasticIP) error { + // This is a new EIP + if a == nil { + // No logic for EIPs - they are just created + } + + // This is an existing EIP + // We should never be changing this + if a != nil { + if changes.PublicIP != nil { + return fi.CannotChangeField("PublicIP") + } + if changes.Subnet != nil { + return fi.CannotChangeField("Subnet") + } + if changes.ID != nil { + return fi.CannotChangeField("ID") + } + } return nil } +// Here is where we actually apply changes to AWS func (_ *ElasticIP) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *ElasticIP) error { - var publicIP *string - tagOnResourceID, err := e.findTagOnResourceID(t.Cloud) - if err != nil { - return err - } + var publicIp *string + var eipId *string + // If this is a new ElasticIP if a == nil { - if tagOnResourceID == nil || e.TagUsingKey == nil { - return fmt.Errorf("cannot create ElasticIP without TagOnResource being set (would leak)") - } glog.V(2).Infof("Creating ElasticIP for VPC") request := &ec2.AllocateAddressInput{} @@ -179,19 +183,29 @@ func (_ *ElasticIP) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *ElasticIP) e e.ID = response.AllocationId e.PublicIP = response.PublicIp - publicIP = response.PublicIp - } else { - publicIP = a.PublicIP + publicIp = e.PublicIP + eipId = response.AllocationId + }else { + publicIp = a.PublicIP + eipId = a.ID } - if publicIP != nil && e.TagUsingKey != nil && tagOnResourceID != nil { - tags := map[string]string{ - *e.TagUsingKey: *publicIP, - } - err := t.AddAWSTags(*tagOnResourceID, tags) - if err != nil { - return fmt.Errorf("error adding tags to resource for ElasticIP: %v", err) - } + + // Tag the associated subnet + if e.Subnet == nil { + return fmt.Errorf("Subnet not set") + } else if e.Subnet.ID == nil { + return fmt.Errorf("Subnet ID not set") + } + tags := make(map[string]string) + tags["AssociatedElasticIp"] = *publicIp + tags["AssociatedElasticIpAllocationId"] = *eipId // Leaving this in for reference, even though we don't use it + err := t.AddAWSTags(*e.Subnet.ID, tags) + if err != nil { + return fmt.Errorf("Unable to tag subnet %v", err) } return nil } + + +// TODO Kris - We need to support EIP for Terraform diff --git a/upup/pkg/fi/cloudup/awstasks/load_balancer_attachment.go b/upup/pkg/fi/cloudup/awstasks/load_balancer_attachment.go index cf5151c1ab0f3..7d2e6d82ad60a 100644 --- a/upup/pkg/fi/cloudup/awstasks/load_balancer_attachment.go +++ b/upup/pkg/fi/cloudup/awstasks/load_balancer_attachment.go @@ -24,21 +24,38 @@ import ( "github.com/golang/glog" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "github.com/aws/aws-sdk-go/service/elb" ) type LoadBalancerAttachment struct { + Name *string LoadBalancer *LoadBalancer + + // LoadBalancerAttachments now support ASGs or direct instances AutoscalingGroup *AutoscalingGroup -} + Subnet *Subnet -func (e *LoadBalancerAttachment) String() string { - return fi.TaskAsString(e) + // Here be dragons.. + // This will *NOT* unmarshal.. for some reason this pointer is initiated as nil + // instead of a pointer to Instance with nil members.. + Instance *Instance } func (e *LoadBalancerAttachment) Find(c *fi.Context) (*LoadBalancerAttachment, error) { cloud := c.Cloud.(awsup.AWSCloud) - if e.AutoscalingGroup != nil { + // Instance only + if e.Instance != nil && e.AutoscalingGroup == nil { + i, err := e.Instance.Find(c) + if err != nil { + return nil, fmt.Errorf("unable to find instance: %v", err) + } + actual := &LoadBalancerAttachment{} + actual.LoadBalancer = e.LoadBalancer + actual.Instance = i + return actual, nil + // ASG only + } else if e.AutoscalingGroup != nil && e.Instance == nil { g, err := findAutoscalingGroup(cloud, *e.AutoscalingGroup.Name) if err != nil { return nil, err @@ -57,6 +74,9 @@ func (e *LoadBalancerAttachment) Find(c *fi.Context) (*LoadBalancerAttachment, e actual.AutoscalingGroup = e.AutoscalingGroup return actual, nil } + } else { + // Invalid request + return nil, fmt.Errorf("Must specify either an instance or an ASG") } return nil, nil @@ -79,16 +99,29 @@ func (s *LoadBalancerAttachment) CheckChanges(a, e, changes *LoadBalancerAttachm } func (_ *LoadBalancerAttachment) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *LoadBalancerAttachment) error { - request := &autoscaling.AttachLoadBalancersInput{} - request.AutoScalingGroupName = e.AutoscalingGroup.Name - request.LoadBalancerNames = []*string{e.LoadBalancer.ID} - - glog.V(2).Infof("Attaching autoscaling group %q to ELB %q", *e.AutoscalingGroup.Name, *e.LoadBalancer.Name) - _, err := t.Cloud.Autoscaling().AttachLoadBalancers(request) - if err != nil { - return fmt.Errorf("error attaching autoscaling group to ELB: %v", err) + if e.AutoscalingGroup != nil && e.Instance == nil { + request := &autoscaling.AttachLoadBalancersInput{} + request.AutoScalingGroupName = e.AutoscalingGroup.Name + request.LoadBalancerNames = []*string{e.LoadBalancer.ID} + glog.V(2).Infof("Attaching autoscaling group %q to ELB %q", *e.AutoscalingGroup.Name, *e.LoadBalancer.Name) + _, err := t.Cloud.Autoscaling().AttachLoadBalancers(request) + if err != nil { + return fmt.Errorf("error attaching autoscaling group to ELB: %v", err) + } + } else if e.AutoscalingGroup == nil && e.Instance != nil { + request := &elb.RegisterInstancesWithLoadBalancerInput{} + var instances []*elb.Instance + i := &elb.Instance{ + InstanceId: e.Instance.ID, + } + instances = append(instances, i) + request.Instances = instances + _, err := t.Cloud.ELB().RegisterInstancesWithLoadBalancer(request) + glog.V(2).Infof("Attaching instance %q to ELB %q", *e.AutoscalingGroup.Name, *e.LoadBalancer.Name) + if err != nil { + return fmt.Errorf("error attaching instance to ELB: %v", err) + } } - return nil } diff --git a/upup/pkg/fi/cloudup/awstasks/natgateway.go b/upup/pkg/fi/cloudup/awstasks/natgateway.go new file mode 100644 index 0000000000000..774153c3b4629 --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/natgateway.go @@ -0,0 +1,185 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/golang/glog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + //"k8s.io/kops/upup/pkg/fi/cloudup/terraform" + "fmt" +) + +//go:generate fitask -type=NatGateway +type NatGateway struct { + Name *string + ElasticIp *ElasticIP + Subnet *Subnet + ID *string +} + +var _ fi.CompareWithID = &NatGateway{} // Validate the IDs + +func (e *NatGateway) CompareWithID() *string { + return e.ID +} + +func (e *NatGateway) Find(c *fi.Context) (*NatGateway, error) { + cloud := c.Cloud.(awsup.AWSCloud) + ID := e.ID + ElasticIp := e.ElasticIp + Subnet := e.Subnet + + // Find via tag on foreign resource + if ID == nil && ElasticIp == nil && Subnet != nil { + var filters []*ec2.Filter + filters = append(filters, awsup.NewEC2Filter("key", "AssociatedNatgateway")) + filters = append(filters, awsup.NewEC2Filter("resource-id", *e.Subnet.ID)) + + request := &ec2.DescribeTagsInput{ + Filters: filters, + } + + response, err := cloud.EC2().DescribeTags(request) + if err != nil { + return nil, fmt.Errorf("error listing tags: %v", err) + } + + if response == nil || len(response.Tags) == 0 { + return nil, nil + } + + if len(response.Tags) != 1 { + return nil, fmt.Errorf("found multiple tags for: %v", e) + } + t := response.Tags[0] + ID = t.Value + glog.V(2).Infof("Found nat gateway via tag: %v", *ID) + } + + if ID != nil { + request := &ec2.DescribeNatGatewaysInput{} + request.NatGatewayIds = []*string{ID} + response, err := cloud.EC2().DescribeNatGateways(request) + if err != nil { + return nil, fmt.Errorf("error listing NAT Gateways: %v", err) + } + + if response == nil || len(response.NatGateways) == 0 { + glog.V(2).Infof("Unable to find Nat Gateways") + return nil, nil + } + if len(response.NatGateways) != 1 { + return nil, fmt.Errorf("found multiple NAT Gateways for: %v", e) + } + a := response.NatGateways[0] + actual := &NatGateway{ + ID: a.NatGatewayId, + } + actual.Subnet = e.Subnet + actual.ElasticIp = e.ElasticIp + e.ID = actual.ID + return actual, nil + } + return nil, nil +} + +func (s *NatGateway) CheckChanges(a, e, changes *NatGateway) error { + + // New + if a == nil { + if e.ElasticIp == nil { + return fi.RequiredField("ElasticIp") + } + if e.Subnet == nil { + return fi.RequiredField("Subnet") + } + } + + // Delta + if a != nil { + if changes.ElasticIp != nil { + return fi.CannotChangeField("ElasticIp") + } + if changes.Subnet != nil { + return fi.CannotChangeField("Subnet") + } + if changes.ID != nil { + return fi.CannotChangeField("ID") + } + } + return nil +} + +func (e *NatGateway) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (_ *NatGateway) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *NatGateway) error { + + // New NGW + var id *string + if a == nil { + glog.V(2).Infof("Creating Nat Gateway") + + request := &ec2.CreateNatGatewayInput{} + request.AllocationId = e.ElasticIp.ID + request.SubnetId = e.Subnet.ID + response, err := t.Cloud.EC2().CreateNatGateway(request) + if err != nil { + return fmt.Errorf("Error creating Nat Gateway: %v", err) + } + e.ID = response.NatGateway.NatGatewayId + id = e.ID + } else { + id = a.ID + } + + // Tag the associated subnet + if e.Subnet == nil { + return fmt.Errorf("Subnet not set") + } else if e.Subnet.ID == nil { + return fmt.Errorf("Subnet ID not set") + } + tags := make(map[string]string) + tags["AssociatedNatgateway"] = *id + err := t.AddAWSTags(*e.Subnet.ID, tags) + if err != nil { + return fmt.Errorf("Unable to tag subnet %v", err) + } + return nil +} + +// TODO Kris - We need to support NGW for Terraform + +//type terraformNATGateway struct { +// AllocationId *string `json:"AllocationID,omitempty"` +// SubnetID *bool `json:"SubnetID,omitempty"` +//} +// +//func (_ *NATGateway) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *NATGateway) error { +// // cloud := t.Cloud.(awsup.AWSCloud) +// +// tf := &terraformNatGateway{ +// AllocationId: e.AllocationID, +// //SubnetID: e.SubnetID, +// } +// +// return t.RenderResource("aws_natgateway", *e.AllocationID, tf) +//} +// +//func (e *NATGateway) TerraformLink() *terraform.Literal { +// return terraform.LiteralProperty("aws_natgateway", *e.AllocationID, "id") +//} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/awstasks/natgateway_fitask.go b/upup/pkg/fi/cloudup/awstasks/natgateway_fitask.go new file mode 100644 index 0000000000000..51b46ecbc1c56 --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/natgateway_fitask.go @@ -0,0 +1,57 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=NatGateway"; DO NOT EDIT + +package awstasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// JSON marshalling boilerplate +type realNatGateway NatGateway + +func (o *NatGateway) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realNatGateway + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = NatGateway(r) + return nil +} + +var _ fi.HasName = &NatGateway{} + +func (e *NatGateway) GetName() *string { + return e.Name +} + +func (e *NatGateway) SetName(name string) { + e.Name = &name +} + +func (e *NatGateway) String() string { + return fi.TaskAsString(e) +} diff --git a/upup/pkg/fi/cloudup/awstasks/route.go b/upup/pkg/fi/cloudup/awstasks/route.go index cec538d7c5dee..2cac43e1b91c3 100644 --- a/upup/pkg/fi/cloudup/awstasks/route.go +++ b/upup/pkg/fi/cloudup/awstasks/route.go @@ -32,9 +32,13 @@ type Route struct { Name *string RouteTable *RouteTable - InternetGateway *InternetGateway Instance *Instance CIDR *string + + // Either an InternetGateway or a NAT Gateway + // MUST be provided. + InternetGateway *InternetGateway + NatGateway *NatGateway } func (e *Route) Find(c *fi.Context) (*Route, error) { @@ -76,6 +80,9 @@ func (e *Route) Find(c *fi.Context) (*Route, error) { if r.GatewayId != nil { actual.InternetGateway = &InternetGateway{ID: r.GatewayId} } + if r.NatGatewayId != nil { + actual.NatGateway = &NatGateway{ID: r.NatGatewayId} + } if r.InstanceId != nil { actual.Instance = &Instance{ID: r.InstanceId} } @@ -115,11 +122,14 @@ func (s *Route) CheckChanges(a, e, changes *Route) error { if e.Instance != nil { targetCount++ } + if e.NatGateway != nil { + targetCount++ + } if targetCount == 0 { - return fmt.Errorf("InternetGateway or Instance is required") + return fmt.Errorf("InternetGateway or Instance or NatGateway is required") } if targetCount != 1 { - return fmt.Errorf("Cannot set both InternetGateway and Instance") + return fmt.Errorf("Cannot set more than 1 InternetGateway or Instance or NatGateway") } } @@ -140,8 +150,12 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error { request.RouteTableId = checkNotNil(e.RouteTable.ID) request.DestinationCidrBlock = checkNotNil(e.CIDR) - if e.InternetGateway != nil { + if e.InternetGateway == nil && e.NatGateway == nil { + return fmt.Errorf("missing target for route") + }else if e.InternetGateway != nil { request.GatewayId = checkNotNil(e.InternetGateway.ID) + }else if e.NatGateway != nil { + request.NatGatewayId = checkNotNil(e.NatGateway.ID) } if e.Instance != nil { @@ -163,8 +177,12 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error { request.RouteTableId = checkNotNil(e.RouteTable.ID) request.DestinationCidrBlock = checkNotNil(e.CIDR) - if e.InternetGateway != nil { + if e.InternetGateway == nil && e.NatGateway == nil { + return fmt.Errorf("missing target for route") + }else if e.InternetGateway != nil { request.GatewayId = checkNotNil(e.InternetGateway.ID) + }else if e.NatGateway != nil { + request.NatGatewayId = checkNotNil(e.NatGateway.ID) } if e.Instance != nil { @@ -194,6 +212,7 @@ type terraformRoute struct { CIDR *string `json:"destination_cidr_block,omitempty"` InternetGatewayID *terraform.Literal `json:"gateway_id,omitempty"` InstanceID *terraform.Literal `json:"instance_id,omitempty"` + // TODO Kris - Add terraform support for NAT Gateway routes } func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Route) error { diff --git a/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go b/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go index de97fb9551eae..397dc07db0898 100644 --- a/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go +++ b/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go @@ -145,6 +145,9 @@ func (e *SecurityGroupRule) matches(rule *ec2.IpPermission) bool { // TODO: Only if len 1? match := false for _, spec := range rule.UserIdGroupPairs { + if e.SourceGroup == nil { + continue + } if aws.StringValue(spec.GroupId) == *e.SourceGroup.ID { match = true break diff --git a/upup/pkg/fi/cloudup/awsup/aws_topology_test.go b/upup/pkg/fi/cloudup/awsup/aws_topology_test.go new file mode 100644 index 0000000000000..9d9af8405a943 --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_topology_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "testing" +) + +// Private Networking +// Integration +// Happy path +func TestValidatePrivateNetworkingHappy(t *testing.T) { + +} + +// Private Networking +// Integration +// Sad path +func TestValidatePrivateNetworkingSad(t *testing.T) { + +} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/loader.go b/upup/pkg/fi/cloudup/loader.go index c0d1991d5b43b..8b7fc49714b61 100644 --- a/upup/pkg/fi/cloudup/loader.go +++ b/upup/pkg/fi/cloudup/loader.go @@ -351,7 +351,6 @@ func (l *Loader) loadObjectMap(key string, data map[string]interface{}) (map[str return nil, fmt.Errorf("cannot determine type for %q", k) } } - t, found := l.typeMap[typeId] if !found { return nil, fmt.Errorf("unknown type %q (in %q)", typeId, key) diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index c130876146d9c..a58adb91b814e 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ + package cloudup import ( @@ -73,6 +74,17 @@ func PopulateClusterSpec(cluster *api.Cluster) (*api.Cluster, error) { return c.fullCluster, nil } + +// +// Here be dragons +// +// This function has some `interesting` things going on. +// In an effort to let the cluster.Spec fall through I am +// hard coding topology in two places.. It seems and feels +// very wrong.. but at least now my new cluster.Spec.Topology +// struct is falling through.. +// @kris-nova +// func (c *populateClusterSpec) run() error { err := c.InputCluster.Validate(false) if err != nil { @@ -81,6 +93,7 @@ func (c *populateClusterSpec) run() error { // Copy cluster & instance groups, so we can modify them freely cluster := &api.Cluster{} + utils.JsonMergeStruct(cluster, c.InputCluster) err = c.assignSubnets(cluster) @@ -104,6 +117,7 @@ func (c *populateClusterSpec) run() error { clusterZones[z.Name] = z } + // Check etcd configuration { for i, etcd := range cluster.Spec.EtcdClusters { @@ -150,6 +164,7 @@ func (c *populateClusterSpec) run() error { } } + keyStore, err := registry.KeyStore(cluster) if err != nil { return err @@ -198,12 +213,18 @@ func (c *populateClusterSpec) run() error { glog.V(2).Infof("Normalizing kubernetes version: %q -> %q", cluster.Spec.KubernetesVersion, versionWithoutV) cluster.Spec.KubernetesVersion = versionWithoutV } - cloud, err := BuildCloud(cluster) if err != nil { return err } + // Hard coding topology here + // + // We want topology to pass through + // Otherwise we were losing the pointer + cluster.Spec.Topology = c.InputCluster.Spec.Topology + + if cluster.Spec.DNSZone == "" { dns, err := cloud.DNS() if err != nil { @@ -216,8 +237,8 @@ func (c *populateClusterSpec) run() error { glog.Infof("Defaulting DNS zone to: %s", dnsZone) cluster.Spec.DNSZone = dnsZone } - tags, err := buildCloudupTags(cluster) + if err != nil { return err } @@ -235,11 +256,14 @@ func (c *populateClusterSpec) run() error { OptionsLoader: loader.NewOptionsLoader(templateFunctions), Tags: tags, } + completed, err := specBuilder.BuildCompleteSpec(&cluster.Spec, c.ModelStore, c.Models) if err != nil { return fmt.Errorf("error building complete spec: %v", err) } + completed.Topology = c.InputCluster.Spec.Topology + fullCluster := &api.Cluster{} *fullCluster = *cluster fullCluster.Spec = *completed @@ -251,7 +275,6 @@ func (c *populateClusterSpec) run() error { } c.fullCluster = fullCluster - return nil } diff --git a/upup/pkg/fi/cloudup/populatecluster_test.go b/upup/pkg/fi/cloudup/populatecluster_test.go index aa9a806f90677..b964ff238ffd3 100644 --- a/upup/pkg/fi/cloudup/populatecluster_test.go +++ b/upup/pkg/fi/cloudup/populatecluster_test.go @@ -33,6 +33,11 @@ func buildMinimalCluster() *api.Cluster { {Name: "us-mock-1b", CIDR: "172.20.2.0/24"}, {Name: "us-mock-1c", CIDR: "172.20.3.0/24"}, } + // Default to public topology + c.Spec.Topology = &api.TopologySpec{ + Masters: api.TopologyPublic, + Nodes: api.TopologyPublic, + } c.Spec.NetworkCIDR = "172.20.0.0/16" c.Spec.NonMasqueradeCIDR = "100.64.0.0/10" c.Spec.CloudProvider = "aws" @@ -283,6 +288,27 @@ func TestPopulateCluster_CloudProvider_Required(t *testing.T) { expectErrorFromPopulateCluster(t, c, "CloudProvider") } +func TestPopulateCluster_TopologyInvalidNil_Required(t *testing.T) { + c := buildMinimalCluster() + c.Spec.Topology.Masters = "" + c.Spec.Topology.Nodes = "" + expectErrorFromPopulateCluster(t, c, "Topology") +} + +func TestPopulateCluster_TopologyInvalidValue_Required(t *testing.T) { + c := buildMinimalCluster() + c.Spec.Topology.Masters = "123" + c.Spec.Topology.Nodes = "abc" + expectErrorFromPopulateCluster(t, c, "Topology") +} + +func TestPopulateCluster_TopologyInvalidMatchingValues_Required(t *testing.T) { + c := buildMinimalCluster() + c.Spec.Topology.Masters = api.TopologyPublic + c.Spec.Topology.Nodes = api.TopologyPrivate + expectErrorFromPopulateCluster(t, c, "Topology") +} + func expectErrorFromPopulateCluster(t *testing.T, c *api.Cluster, message string) { _, err := PopulateClusterSpec(c) if err == nil { diff --git a/upup/pkg/fi/cloudup/tagbuilder.go b/upup/pkg/fi/cloudup/tagbuilder.go index c9cf0b39e3917..6fa6bf02a0599 100644 --- a/upup/pkg/fi/cloudup/tagbuilder.go +++ b/upup/pkg/fi/cloudup/tagbuilder.go @@ -14,6 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ +/****************************************************************************** +* The Kops Tag Builder +* +* Tags are how we manage kops functionality. +* +******************************************************************************/ + package cloudup import ( @@ -23,6 +30,8 @@ import ( "k8s.io/kops/upup/pkg/fi" ) +// +// func buildCloudupTags(cluster *api.Cluster) (map[string]struct{}, error) { // TODO: Make these configurable? useMasterASG := true @@ -54,10 +63,22 @@ func buildCloudupTags(cluster *api.Cluster) (map[string]struct{}, error) { if useMasterLB { tags["_master_lb"] = struct{}{} - } else { + } else if cluster.Spec.Topology.Masters == api.TopologyPublic { tags["_not_master_lb"] = struct{}{} } + // Network Topologies + if cluster.Spec.Topology == nil { + return nil, fmt.Errorf("missing topology spec") + } + if cluster.Spec.Topology.Masters == api.TopologyPublic && cluster.Spec.Topology.Nodes == api.TopologyPublic { + tags["_topology_public"] = struct{}{} + } else if cluster.Spec.Topology.Masters == api.TopologyPrivate && cluster.Spec.Topology.Nodes == api.TopologyPrivate { + tags["_topology_private"] = struct{}{} + } else { + return nil, fmt.Errorf("Unable to parse topology. Unsupported topology configuration. Masters and nodes must match!") + } + if fi.BoolValue(cluster.Spec.IsolateMasters) { tags["_isolate_masters"] = struct{}{} } diff --git a/upup/pkg/fi/cloudup/tagbuilder_test.go b/upup/pkg/fi/cloudup/tagbuilder_test.go index 5276ea833645f..425b775c06040 100644 --- a/upup/pkg/fi/cloudup/tagbuilder_test.go +++ b/upup/pkg/fi/cloudup/tagbuilder_test.go @@ -54,6 +54,10 @@ func buildCluster(clusterArgs interface{}) *api.Cluster { KubernetesVersion: cParams.KubernetesVersion, Networking: networking, UpdatePolicy: fi.String(cParams.UpdatePolicy), + Topology: &api.TopologySpec{ + Masters: api.TopologyPublic, + Nodes: api.TopologyPublic, + }, }, } } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index afe1dbefd3d7b..5fa34f6b23863 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -14,6 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ +/****************************************************************************** +Template Functions are what map functions in the models, to internal logic in +kops. This is the point where we connect static YAML configuration to dynamic +runtime values in memory. + +When defining a new function: + - Build the new function here + - Define the new function in AddTo() + dest["MyNewFunction"] = MyNewFunction // <-- Function Pointer +******************************************************************************/ + package cloudup import ( @@ -71,10 +82,22 @@ func (tf *TemplateFunctions) WellKnownServiceIP(id int) (net.IP, error) { return nil, fmt.Errorf("Unexpected IP address type for ServiceClusterIPRange: %s", tf.cluster.Spec.ServiceClusterIPRange) } +// This will define the available functions we can use in our YAML models +// If we are trying to get a new function implemented it MUST +// be defined here. func (tf *TemplateFunctions) AddTo(dest template.FuncMap) { dest["EtcdClusterMemberTags"] = tf.EtcdClusterMemberTags dest["SharedVPC"] = tf.SharedVPC + // Network topology definitions + dest["IsTopologyPublic"] = tf.IsTopologyPublic + dest["IsTopologyPrivate"] = tf.IsTopologyPrivate + dest["IsTopologyPrivateMasters"] = tf.IsTopologyPrivateMasters + dest["WithBastion"] = tf.WithBastion + dest["GetBastionImageId"] = tf.GetBastionImageId + dest["GetBastionZone"] = tf.GetBastionZone + dest["GetELBName32"] = tf.GetELBName32 + dest["SharedZone"] = tf.SharedZone dest["WellKnownServiceIP"] = tf.WellKnownServiceIP dest["AdminCIDR"] = tf.AdminCIDR @@ -150,6 +173,61 @@ func (tf *TemplateFunctions) SharedVPC() bool { return tf.cluster.SharedVPC() } +// These are the network topology functions. They are boolean logic for checking which type of +// topology this cluster is set to be deployed with. +func (tf *TemplateFunctions) IsTopologyPrivate() bool { return tf.cluster.IsTopologyPrivate() } +func (tf *TemplateFunctions) IsTopologyPublic() bool { return tf.cluster.IsTopologyPublic() } +func (tf *TemplateFunctions) IsTopologyPrivateMasters() bool { return tf.cluster.IsTopologyPrivateMasters() } + +func (tf *TemplateFunctions) WithBastion() bool { + return !tf.cluster.Spec.Topology.BypassBastion +} + +// This function is replacing existing yaml +func (tf *TemplateFunctions) GetBastionZone() (string, error) { + var name string + if len(tf.cluster.Spec.Zones) <= 1 { + return "", fmt.Errorf("Unable to detect zone name for bastion") + } else { + // If we have a list, always use the first one + name = tf.cluster.Spec.Zones[0].Name + } + return name, nil +} + +// Will attempt to calculate a meaningful name for an ELB given a prefix +// Will never return a string longer than 32 chars +func (tf *TemplateFunctions) GetELBName32(prefix string) (string, error ){ + var returnString string + c := tf.cluster.Name + s := strings.Split(c, ".") + if len(s) > 0 { + returnString = fmt.Sprintf("%s-%s", prefix, s[0]) + }else { + returnString = fmt.Sprintf("%s-%s", prefix, c) + } + if len(returnString) > 32 { + returnString = returnString[:32] + } + return returnString, nil +} + +func (tf *TemplateFunctions) GetBastionImageId() (string, error) { + if len(tf.instanceGroups) == 0 { + return "", fmt.Errorf("Unable to find AMI in instance group") + }else if len(tf.instanceGroups) > 0 { + ami := tf.instanceGroups[0].Spec.Image + for i := 1; i < len(tf.instanceGroups); i++ { + // If we can't be sure all AMIs are the same, we don't know which one to use for the bastion host + if tf.instanceGroups[i].Spec.Image != ami { + return "", fmt.Errorf("Unable to use multiple image id's with a private bastion") + } + } + return ami, nil; + } + return "", nil +} + // SharedZone is a simple helper function which makes the templates for a shared Zone clearer func (tf *TemplateFunctions) SharedZone(zone *api.ClusterZoneSpec) bool { return zone.ProviderID != "" @@ -298,4 +376,4 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) { argv = append(argv, "-v=8") return argv, nil -} +} \ No newline at end of file diff --git a/upup/pkg/fi/fitasks/managedfile_fitask.go b/upup/pkg/fi/fitasks/managedfile_fitask.go new file mode 100644 index 0000000000000..950db7f286785 --- /dev/null +++ b/upup/pkg/fi/fitasks/managedfile_fitask.go @@ -0,0 +1,59 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=ManagedFile"; DO NOT EDIT + +package fitasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// ManagedFile + +// JSON marshalling boilerplate +type realManagedFile ManagedFile + +func (o *ManagedFile) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realManagedFile + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = ManagedFile(r) + return nil +} + +var _ fi.HasName = &ManagedFile{} + +func (e *ManagedFile) GetName() *string { + return e.Name +} + +func (e *ManagedFile) SetName(name string) { + e.Name = &name +} + +func (e *ManagedFile) String() string { + return fi.TaskAsString(e) +} diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index e4c9b78bb7586..4d4a5bab6e78d 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -111,6 +111,11 @@ func newTemplateFunctions(nodeupConfig *NodeUpConfig, cluster *api.Cluster, inst } func (t *templateFunctions) populate(dest template.FuncMap) { + + + dest["IsTopologyPublic"] = t.cluster.IsTopologyPublic + dest["IsTopologyPrivate"] = t.cluster.IsTopologyPrivate + dest["CACertificatePool"] = t.CACertificatePool dest["CACertificate"] = t.CACertificate dest["PrivateKey"] = t.PrivateKey diff --git a/upup/pkg/kutil/delete_cluster.go b/upup/pkg/kutil/delete_cluster.go index 31ca26f7b2b70..b24bff738563a 100644 --- a/upup/pkg/kutil/delete_cluster.go +++ b/upup/pkg/kutil/delete_cluster.go @@ -53,19 +53,19 @@ type DeleteCluster struct { } type ResourceTracker struct { - Name string - Type string - ID string + Name string + Type string + ID string - blocks []string - blocked []string - done bool + blocks []string + blocked []string + done bool deleter func(cloud fi.Cloud, tracker *ResourceTracker) error groupKey string groupDeleter func(cloud fi.Cloud, trackers []*ResourceTracker) error - obj interface{} + obj interface{} } type listFn func(fi.Cloud, string) ([]*ResourceTracker, error) @@ -91,7 +91,7 @@ func buildEC2Filters(cloud fi.Cloud) []*ec2.Filter { var filters []*ec2.Filter for k, v := range tags { - filter := awsup.NewEC2Filter("tag:"+k, v) + filter := awsup.NewEC2Filter("tag:" + k, v) filters = append(filters, filter) } return filters @@ -131,7 +131,7 @@ func (c *DeleteCluster) ListResources() (map[string]*ResourceTracker, error) { return nil, err } for _, t := range trackers { - resources[t.Type+":"+t.ID] = t + resources[t.Type + ":" + t.ID] = t } } @@ -152,8 +152,8 @@ func (c *DeleteCluster) ListResources() (map[string]*ResourceTracker, error) { if vpcID == "" || igwID == "" { continue } - if resources["vpc:"+vpcID] != nil && resources["internet-gateway:"+igwID] == nil { - resources["internet-gateway:"+igwID] = &ResourceTracker{ + if resources["vpc:" + vpcID] != nil && resources["internet-gateway:" + igwID] == nil { + resources["internet-gateway:" + igwID] = &ResourceTracker{ Name: FindName(igw.Tags), ID: igwID, Type: "internet-gateway", @@ -192,7 +192,7 @@ func addUntaggedRouteTables(cloud awsup.AWSCloud, clusterName string, resources continue } - if resources["vpc:"+vpcID] == nil { + if resources["vpc:" + vpcID] == nil { // Not deleting this VPC; ignore continue } @@ -215,8 +215,8 @@ func addUntaggedRouteTables(cloud awsup.AWSCloud, clusterName string, resources } t := buildTrackerForRouteTable(rt) - if resources[t.Type+":"+t.ID] == nil { - resources[t.Type+":"+t.ID] = t + if resources[t.Type + ":" + t.ID] == nil { + resources[t.Type + ":" + t.ID] = t } } @@ -252,7 +252,6 @@ func (c *DeleteCluster) DeleteResources(resources map[string]*ResourceTracker) e iterationsWithNoProgress := 0 for { // TODO: Some form of default ordering based on types? - // TODO: Give up eventually? failed := make(map[string]*ResourceTracker) @@ -365,7 +364,7 @@ func (c *DeleteCluster) DeleteResources(resources map[string]*ResourceTracker) e } iterationsWithNoProgress++ - if iterationsWithNoProgress > 30 { + if iterationsWithNoProgress > 42 { return fmt.Errorf("Not making progress deleting resources; giving up") } @@ -470,19 +469,19 @@ func ListInstances(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, erro } var blocks []string - blocks = append(blocks, "vpc:"+aws.StringValue(instance.VpcId)) + blocks = append(blocks, "vpc:" + aws.StringValue(instance.VpcId)) for _, volume := range instance.BlockDeviceMappings { if volume.Ebs == nil { continue } - blocks = append(blocks, "volume:"+aws.StringValue(volume.Ebs.VolumeId)) + blocks = append(blocks, "volume:" + aws.StringValue(volume.Ebs.VolumeId)) } for _, sg := range instance.SecurityGroups { - blocks = append(blocks, "security-group:"+aws.StringValue(sg.GroupId)) + blocks = append(blocks, "security-group:" + aws.StringValue(sg.GroupId)) } - blocks = append(blocks, "subnet:"+aws.StringValue(instance.SubnetId)) - blocks = append(blocks, "vpc:"+aws.StringValue(instance.VpcId)) + blocks = append(blocks, "subnet:" + aws.StringValue(instance.SubnetId)) + blocks = append(blocks, "vpc:" + aws.StringValue(instance.VpcId)) tracker.blocks = blocks @@ -567,7 +566,7 @@ func ListSecurityGroups(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, } var blocks []string - blocks = append(blocks, "vpc:"+aws.StringValue(sg.VpcId)) + blocks = append(blocks, "vpc:" + aws.StringValue(sg.VpcId)) tracker.blocks = blocks @@ -731,8 +730,8 @@ func ListKeypairs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error glog.V(2).Infof("Listing EC2 Keypairs") request := &ec2.DescribeKeyPairsInput{ - // We need to match both the name and a prefix - //Filters: []*ec2.Filter{awsup.NewEC2Filter("key-name", keypairName)}, + // We need to match both the name and a prefix + //Filters: []*ec2.Filter{awsup.NewEC2Filter("key-name", keypairName)}, } response, err := c.EC2().DescribeKeyPairs(request) if err != nil { @@ -743,7 +742,7 @@ func ListKeypairs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error for _, keypair := range response.KeyPairs { name := aws.StringValue(keypair.KeyName) - if name != keypairName && !strings.HasPrefix(name, keypairName+"-") { + if name != keypairName && !strings.HasPrefix(name, keypairName + "-") { continue } tracker := &ResourceTracker{ @@ -792,13 +791,15 @@ func DeleteSubnet(cloud fi.Cloud, tracker *ResourceTracker) error { } func ListSubnets(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { + c := cloud.(awsup.AWSCloud) subnets, err := DescribeSubnets(cloud) if err != nil { return nil, fmt.Errorf("error listing subnets: %v", err) } var trackers []*ResourceTracker - + elasticIPs := make(map[string]bool) + ngws := make(map[string]bool) for _, subnet := range subnets { tracker := &ResourceTracker{ Name: FindName(subnet.Tags), @@ -807,12 +808,86 @@ func ListSubnets(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) deleter: DeleteSubnet, } + // Get tags and append with EIPs/NGWs as needed + for _, tag := range subnet.Tags { + name := aws.StringValue(tag.Key) + ip := "" + if name == "AssociatedElasticIp" { + ip = aws.StringValue(tag.Value) + } + if ip != "" { + elasticIPs[ip] = true + } + id := "" + if name == "AssociatedNatgateway" { + id = aws.StringValue(tag.Value) + } + if id != "" { + ngws[id] = true + } + } + var blocks []string - blocks = append(blocks, "vpc:"+aws.StringValue(subnet.VpcId)) + blocks = append(blocks, "vpc:" + aws.StringValue(subnet.VpcId)) tracker.blocks = blocks trackers = append(trackers, tracker) + + // Associated Elastic IPs + if len(elasticIPs) != 0 { + glog.V(2).Infof("Querying EC2 Elastic IPs") + request := &ec2.DescribeAddressesInput{} + response, err := c.EC2().DescribeAddresses(request) + if err != nil { + return nil, fmt.Errorf("error describing addresses: %v", err) + } + + for _, address := range response.Addresses { + ip := aws.StringValue(address.PublicIp) + if !elasticIPs[ip] { + continue + } + + tracker := &ResourceTracker{ + Name: ip, + ID: aws.StringValue(address.AllocationId), + Type: "elastic-ip", + deleter: DeleteElasticIP, + } + + trackers = append(trackers, tracker) + + } + } + + // Associated Nat Gateways + if len(ngws) != 0 { + glog.V(2).Infof("Querying Nat Gateways") + request := &ec2.DescribeNatGatewaysInput{} + response, err := c.EC2().DescribeNatGateways(request) + if err != nil { + return nil, fmt.Errorf("error describing nat gateways: %v", err) + } + + for _, ngw := range response.NatGateways { + id := aws.StringValue(ngw.NatGatewayId) + if !ngws[id] { + continue + } + + tracker := &ResourceTracker{ + Name: id, + ID: aws.StringValue(ngw.NatGatewayId), + Type: "natgateway", + deleter: DeleteNGW, + } + + trackers = append(trackers, tracker) + + } + } + } return trackers, nil @@ -909,10 +984,10 @@ func buildTrackerForRouteTable(rt *ec2.RouteTable) *ResourceTracker { var blocks []string var blocked []string - blocks = append(blocks, "vpc:"+aws.StringValue(rt.VpcId)) + blocks = append(blocks, "vpc:" + aws.StringValue(rt.VpcId)) for _, a := range rt.Associations { - blocked = append(blocked, "subnet:"+aws.StringValue(a.SubnetId)) + blocked = append(blocked, "subnet:" + aws.StringValue(a.SubnetId)) } tracker.blocks = blocks @@ -1064,7 +1139,7 @@ func ListInternetGateways(cloud fi.Cloud, clusterName string) ([]*ResourceTracke var blocks []string for _, a := range o.Attachments { if aws.StringValue(a.VpcId) != "" { - blocks = append(blocks, "vpc:"+aws.StringValue(a.VpcId)) + blocks = append(blocks, "vpc:" + aws.StringValue(a.VpcId)) } } tracker.blocks = blocks @@ -1167,7 +1242,7 @@ func ListVPCs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { } var blocks []string - blocks = append(blocks, "dhcp-options:"+aws.StringValue(v.DhcpOptionsId)) + blocks = append(blocks, "dhcp-options:" + aws.StringValue(v.DhcpOptionsId)) tracker.blocks = blocks @@ -1223,9 +1298,9 @@ func ListAutoScalingGroups(cloud fi.Cloud, clusterName string) ([]*ResourceTrack if subnet == "" { continue } - blocks = append(blocks, "subnet:"+subnet) + blocks = append(blocks, "subnet:" + subnet) } - blocks = append(blocks, TypeAutoscalingLaunchConfig+":"+aws.StringValue(asg.LaunchConfigurationName)) + blocks = append(blocks, TypeAutoscalingLaunchConfig + ":" + aws.StringValue(asg.LaunchConfigurationName)) tracker.blocks = blocks @@ -1387,12 +1462,12 @@ func ListELBs(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error) { var blocks []string for _, sg := range elb.SecurityGroups { - blocks = append(blocks, "security-group:"+aws.StringValue(sg)) + blocks = append(blocks, "security-group:" + aws.StringValue(sg)) } for _, s := range elb.Subnets { - blocks = append(blocks, "subnet:"+aws.StringValue(s)) + blocks = append(blocks, "subnet:" + aws.StringValue(s)) } - blocks = append(blocks, "vpc:"+aws.StringValue(elb.VPCId)) + blocks = append(blocks, "vpc:" + aws.StringValue(elb.VPCId)) tracker.blocks = blocks @@ -1480,6 +1555,25 @@ func DeleteElasticIP(cloud fi.Cloud, t *ResourceTracker) error { return nil } +func DeleteNGW(cloud fi.Cloud, t *ResourceTracker) error { + c := cloud.(awsup.AWSCloud) + + id := t.ID + + glog.V(2).Infof("Removing NGW %s", t.Name) + request := &ec2.DeleteNatGatewayInput{ + NatGatewayId: &id, + } + _, err := c.EC2().DeleteNatGateway(request) + if err != nil { + if IsDependencyViolation(err) { + return err + } + return fmt.Errorf("error deleting ngw %q: %v", t.Name, err) + } + return nil +} + func deleteRoute53Records(cloud fi.Cloud, zone *route53.HostedZone, trackers []*ResourceTracker) error { c := cloud.(awsup.AWSCloud) @@ -1648,8 +1742,8 @@ func ListIAMRoles(cloud fi.Cloud, clusterName string) ([]*ResourceTracker, error c := cloud.(awsup.AWSCloud) remove := make(map[string]bool) - remove["masters."+clusterName] = true - remove["nodes."+clusterName] = true + remove["masters." + clusterName] = true + remove["nodes." + clusterName] = true var roles []*iam.Role // Find roles matching remove map @@ -1725,8 +1819,8 @@ func ListIAMInstanceProfiles(cloud fi.Cloud, clusterName string) ([]*ResourceTra c := cloud.(awsup.AWSCloud) remove := make(map[string]bool) - remove["masters."+clusterName] = true - remove["nodes."+clusterName] = true + remove["masters." + clusterName] = true + remove["nodes." + clusterName] = true var profiles []*iam.InstanceProfile