Topology Support (AKA Private Networking) #428

cmd/kops/create_cluster.go

@@ -58,7 +58,10 @@ type CreateClusterOptions struct {
 	AssociatePublicIP bool
 
 	// Channel is the location of the api.Channel to use for our defaults
-	Channel string
+	Channel           string
+
+	// The network topology to use
+	Topology          string
 }
 
 func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
@@ -111,6 +114,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 
 	cmd.Flags().StringVar(&options.Channel, "channel", api.DefaultChannel, "Channel for default versions and configuration to use")
 
+	// Network topology
+	cmd.Flags().StringVarP(&options.Topology, "topology", "t", "public", "Controls network topology for the cluster. public|private. Default is 'public'.")
+
 	return cmd
 }
 
@@ -359,6 +365,23 @@ func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
 		}
 	}
 
+
+	// Network Topology
+	switch  c.Topology{
+	case api.TopologyPublic:
+		cluster.Spec.Topology = &api.TopologySpec{Masters: api.TopologyPublic, Nodes: api.TopologyPublic, BypassBastion: false}
+	case api.TopologyPrivate:
+		if c.Networking != "cni" {
+			return fmt.Errorf("Invalid networking option %s. Currently only '--networking cni' is supported for private topologies", c.Networking)
+		}
+		cluster.Spec.Topology = &api.TopologySpec{Masters: api.TopologyPrivate, Nodes: api.TopologyPrivate, BypassBastion: false}
+	case "":
+		glog.Warningf("Empty topology. Defaulting to public topology.")
+		cluster.Spec.Topology = &api.TopologySpec{Masters: api.TopologyPublic, Nodes: api.TopologyPublic, BypassBastion: false}
+	default:
+		return fmt.Errorf("Invalid topology %s.", c.Topology)
+	}
+
 	sshPublicKeys := make(map[string][]byte)
 	if c.SSHPublicKey != "" {
 		c.SSHPublicKey = utils.ExpandPath(c.SSHPublicKey)
@@ -505,4 +528,4 @@ func parseZoneList(s string) []string {
 		filtered = append(filtered, v)
 	}
 	return filtered
-}
+}

cmd/kops/update_cluster.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/kops/upup/pkg/kutil"
 	"os"
 	"strings"
+	"k8s.io/kops/pkg/apis/kops"
 )
 
 type UpdateClusterOptions struct {
@@ -188,7 +189,11 @@ func RunUpdateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
 			fmt.Printf("\n")
 			fmt.Printf("Suggestions:\n")
 			fmt.Printf(" * list nodes: kubectl get nodes --show-labels\n")
-			fmt.Printf(" * ssh to the master: ssh -i ~/.ssh/id_rsa admin@%s\n", cluster.Spec.MasterPublicName)
+			if cluster.Spec.Topology.Masters == kops.TopologyPublic {
+				fmt.Printf(" * ssh to the master: ssh -i ~/.ssh/id_rsa admin@%s\n", cluster.Spec.MasterPublicName)
+			}else {
+				fmt.Printf(" * ssh to the bastion: ssh -i ~/.ssh/id_rsa admin@%s\n", cluster.Spec.MasterPublicName)
+			}
 			fmt.Printf(" * read about installing addons: https://github.com/kubernetes/kops/blob/master/docs/addons.md\n")
 			fmt.Printf("\n")
 		}

docs/topology.md

@@ -0,0 +1,45 @@
+# Network Topologies in Kops
+
+Kops supports a number of pre defined network topologies. They are separated into commonly used scenarios, or topologies.
+
+Each of the supported topologies are listed below, with an example on how to deploy them.
+
+# AWS
+
+Kops supports the following topologies on AWS
+
+|      Topology     |   Value    | Description                                                                                                 |
+| ----------------- |----------- | ----------------------------------------------------------------------------------------------------------- |
+|   Public Cluster  |   public   | All masters/nodes will be launched in a **public subnet** in the VPC                                        |
+|   Private Cluster |   private  | All masters/nodes will be launched in a **private subnet** in the VPC                                       |
+
+
+[More information](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html) on Public and Private subnets in AWS
+
+Notes on subnets
+
+##### Public Subnet
+If a subnet's traffic is routed to an Internet gateway, the subnet is known as a public subnet.
+
+##### Private Subnet
+If a subnet doesn't have a route to the Internet gateway, the subnet is known as a private subnet.
+
+Private topologies *will* have public access via the Kubernetes API and an (optional) SSH bastion instance.
+
+# Defining a topology on create
+
+To specify a topology use the `--topology` or `-t` flag as in :
+
+```
+kops create cluster ... --topology public|private
+```
+
+# Troubleshooting
+
+- Right now we require `-networking cni` for all private topologies.
+- Right now a manual install of weave is required for private topologies.
+- Right now upgrading from a public cluster to a private cluster is considered very **experimental**
+
+```
+kubectl create -f https://git.io/weave-kube
+```

pkg/apis/kops/cluster.go

@@ -29,7 +29,7 @@ import (
 
 type Cluster struct {
 	unversioned.TypeMeta `json:",inline"`
-	ObjectMeta    `json:"metadata,omitempty"`
+	ObjectMeta           `json:"metadata,omitempty"`
 
 	Spec ClusterSpec `json:"spec,omitempty"`
 }
@@ -77,6 +77,11 @@ type ClusterSpec struct {
 	// NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC)
 	NetworkID string `json:"networkID,omitempty"`
 
+	// Topology defines the type of network topology to use on the cluster - default public
+	// This is heavily weighted towards AWS for the time being, but should also be agnostic enough
+	// to port out to GCE later if needed
+	Topology *TopologySpec `json:"topology,omitempty"`
+
 	// SecretStore is the VFS path to where secrets are stored
 	SecretStore string `json:"secretStore,omitempty"`
 	// KeyStore is the VFS path to where SSL keys and certificates are stored
@@ -275,7 +280,14 @@ type EtcdMemberSpec struct {
 
 type ClusterZoneSpec struct {
 	Name string `json:"name,omitempty"`
-	CIDR string `json:"cidr,omitempty"`
+
+	// For Private network topologies we need to have 2
+	// CIDR blocks.
+	// 1 - Utility (Public) Subnets
+	// 2 - Operating (Private) Subnets
+
+	PrivateCIDR string `json:"privateCIDR,omitempty"`
+	CIDR        string `json:"cidr,omitempty"`
 
 	// ProviderID is the cloud provider id for the objects associated with the zone (the subnet on AWS)
 	ProviderID string `json:"id,omitempty"`
@@ -341,10 +353,10 @@ func (c *Cluster) FillDefaults() error {
 		// OK
 	} else if c.Spec.Networking.Kubenet != nil {
 		// OK
-	} else if c.Spec.Networking.External != nil {
-		// OK
 	} else if c.Spec.Networking.CNI != nil {
 		// OK
+	} else if c.Spec.Networking.External != nil {
+		// OK
 	} else {
 		// No networking model selected; choose Kubenet
 		c.Spec.Networking.Kubenet = &KubenetNetworkingSpec{}
@@ -414,21 +426,27 @@ func FindLatestKubernetesVersion() (string, error) {
 
 func (z *ClusterZoneSpec) performAssignments(c *Cluster) error {
 	if z.CIDR == "" {
-		cidr, err := z.assignCIDR(c)
+		err := z.assignCIDR(c)
 		if err != nil {
 			return err
 		}
-		glog.Infof("Assigned CIDR %s to zone %s", cidr, z.Name)
-		z.CIDR = cidr
 	}
-
 	return nil
 }
 
-func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) {
+// Will generate a CIDR block based on the last character in
+// the cluster.Spec.Zones structure.
+//
+func (z *ClusterZoneSpec) assignCIDR(c *Cluster) error {
 	// TODO: We probably could query for the existing subnets & allocate appropriately
 	// for now we'll require users to set CIDRs themselves
 
+	// Used in calculating private subnet blocks (if needed only)
+	needsPrivateBlock := false
+	if c.Spec.Topology.Masters == TopologyPrivate || c.Spec.Topology.Nodes == TopologyPrivate {
+		needsPrivateBlock = true
+	}
+
 	lastCharMap := make(map[byte]bool)
 	for _, nodeZone := range c.Spec.Zones {
 		lastChar := nodeZone.Name[len(nodeZone.Name)-1]
@@ -452,13 +470,13 @@ func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) {
 			}
 		}
 		if index == -1 {
-			return "", fmt.Errorf("zone not configured: %q", z.Name)
+			return fmt.Errorf("zone not configured: %q", z.Name)
 		}
 	}
 
 	_, cidr, err := net.ParseCIDR(c.Spec.NetworkCIDR)
 	if err != nil {
-		return "", fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR)
+		return fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR)
 	}
 	networkLength, _ := cidr.Mask.Size()
 
@@ -475,14 +493,49 @@ func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) {
 		subnetIP := make(net.IP, len(ip4))
 		binary.BigEndian.PutUint32(subnetIP, n)
 		subnetCIDR := subnetIP.String() + "/" + strconv.Itoa(networkLength)
+		z.CIDR = subnetCIDR
 		glog.V(2).Infof("Computed CIDR for subnet in zone %q as %q", z.Name, subnetCIDR)
-		return subnetCIDR, nil
+		glog.Infof("Assigned CIDR %s to zone %s", subnetCIDR, z.Name)
+
+		if needsPrivateBlock {
+			m := binary.BigEndian.Uint32(ip4)
+			// All Private CIDR blocks are at the end of our range
+			m += uint32(index+len(c.Spec.Zones)) << uint(32-networkLength)
+			privSubnetIp := make(net.IP, len(ip4))
+			binary.BigEndian.PutUint32(privSubnetIp, m)
+			privCIDR := privSubnetIp.String() + "/" + strconv.Itoa(networkLength)
+			z.PrivateCIDR = privCIDR
+			glog.V(2).Infof("Computed Private CIDR for subnet in zone %q as %q", z.Name, privCIDR)
+			glog.Infof("Assigned Private CIDR %s to zone %s", privCIDR, z.Name)
+		}
+
+		return nil
 	}
 
-	return "", fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR)
+	return fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR)
 }
 
 // SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
 func (c *Cluster) SharedVPC() bool {
 	return c.Spec.NetworkID != ""
 }
+
+// --------------------------------------------------------------------------------------------
+// Network Topology functions for template parsing
+//
+// Each of these functions can be used in the model templates
+// The go template package currently only supports boolean
+// operations, so the logic is mapped here as *Cluster functions.
+//
+// A function will need to be defined for all new topologies, if we plan to use them in the
+// model templates.
+// --------------------------------------------------------------------------------------------
+func (c *Cluster) IsTopologyPrivate() bool {
+	return (c.Spec.Topology.Masters == TopologyPrivate && c.Spec.Topology.Nodes == TopologyPrivate)
+}
+func (c *Cluster) IsTopologyPublic() bool {
+	return (c.Spec.Topology.Masters == TopologyPublic && c.Spec.Topology.Nodes == TopologyPublic)
+}
+func (c *Cluster) IsTopologyPrivateMasters() bool {
+	return (c.Spec.Topology.Masters == TopologyPrivate && c.Spec.Topology.Nodes == TopologyPublic)
+}

pkg/apis/kops/topology.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kops
+
+const (
+	TopologyPublic         = "public"
+	TopologyPrivate        = "private"
+)
+
+type TopologySpec struct {
+	// The environment to launch the Kubernetes masters in public|private
+	Masters       string `json:"masters,omitempty"`
+
+	// The environment to launch the Kubernetes nodes in public|private
+	Nodes         string `json:"nodes,omitempty"`
+
+	// Controls if a private topology should deploy a bastion host or not
+	// The bastion host is designed to be a simple, and secure bridge between
+	// the public subnet and the private subnet
+	BypassBastion bool `json:"bypassBastion,omitempty"`
+}

pkg/apis/kops/v1alpha1/cluster.go

@@ -78,6 +78,11 @@ type ClusterSpec struct {
 	// NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC)
 	NetworkID string `json:"networkID,omitempty"`
 
+	// Topology defines the type of network topology to use on the cluster - default public
+	// This is heavily weighted towards AWS for the time being, but should also be agnostic enough
+	// to port out to GCE later if needed
+	Topology *TopologySpec `json:"topology,omitempty"`
+
 	// SecretStore is the VFS path to where secrets are stored
 	SecretStore string `json:"secretStore,omitempty"`
 	// KeyStore is the VFS path to where SSL keys and certificates are stored

pkg/apis/kops/v1alpha1/topology.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+const (
+	TopologyPublic         = "public"
+	TopologyPrivate        = "private"
+)
+
+type TopologySpec struct {
+	// The environment to launch the Kubernetes masters in public|private
+	Masters       string `json:"masters,omitempty"`
+
+	// The environment to launch the Kubernetes nodes in public|private
+	Nodes         string `json:"nodes,omitempty"`
+
+	// Controls if a private topology should deploy a bastion host or not
+	// The bastion host is designed to be a simple, and secure bridge between
+	// the public subnet and the private subnet
+	BypassBastion bool `json:"bypassBastion,omitempty"`
+}

pkg/apis/kops/validation.go

@@ -292,6 +292,21 @@ func (c *Cluster) Validate(strict bool) error {
 		}
 	}
 
+	// Topology support
+	if c.Spec.Topology.Masters != "" && c.Spec.Topology.Nodes != "" {
+		if c.Spec.Topology.Masters != TopologyPublic && c.Spec.Topology.Masters != TopologyPrivate {
+			return fmt.Errorf("Invalid Masters value for Topology")
+		} else if c.Spec.Topology.Nodes != TopologyPublic && c.Spec.Topology.Nodes != TopologyPrivate {
+			return fmt.Errorf("Invalid Nodes value for Topology")
+		// Until we support other topologies - these must match
+		} else if c.Spec.Topology.Masters != c.Spec.Topology.Nodes {
+			return fmt.Errorf("Topology Nodes must match Topology Masters")
+		}
+
+	}else{
+		return fmt.Errorf("Topology requires non-nil values for Masters and Nodes")
+	}
+
 	// Etcd
 	{
 		if len(c.Spec.EtcdClusters) == 0 {