Merge pull request #4711 from justinsb/shared_network_objects

Fix shared network objects

cmd/kops/integration_test.go

@@ -123,6 +123,11 @@ func TestPrivateKopeio(t *testing.T) {
 	runTestAWS(t, "privatekopeio.example.com", "privatekopeio", "v1alpha2", true, 1)
 }
 
+// TestPrivateSharedSubnet runs the test on a configuration with private topology & shared subnets
+func TestPrivateSharedSubnet(t *testing.T) {
+	runTestAWS(t, "private-shared-subnet.example.com", "private-shared-subnet", "v1alpha2", true, 1)
+}
+
 // TestPrivateDns1 runs the test on a configuration with private topology, private dns
 func TestPrivateDns1(t *testing.T) {
 	runTestAWS(t, "privatedns1.example.com", "privatedns1", "v1alpha2", true, 1)

cmd/kops/lifecycle_integration_test.go

@@ -80,6 +80,22 @@ func TestLifecycleSharedVPC(t *testing.T) {
 	})
 }
 
+// TestLifecycleSharedSubnet runs the test on a shared subnet
+func TestLifecycleSharedSubnet(t *testing.T) {
+	runLifecycleTestAWS(&LifecycleTestOptions{
+		t:      t,
+		SrcDir: "shared_subnet",
+	})
+}
+
+// TestLifecyclePrivateSharedSubnet runs the test on a shared subnet with private topology
+func TestLifecyclePrivateSharedSubnet(t *testing.T) {
+	runLifecycleTestAWS(&LifecycleTestOptions{
+		t:      t,
+		SrcDir: "private-shared-subnet",
+	})
+}
+
 func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptions) {
 	t := o.t
 

pkg/model/network.go

@@ -101,11 +101,18 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	allSubnetsShared := true
+	allSubnetsSharedInZone := make(map[string]bool)
+	for i := range b.Cluster.Spec.Subnets {
+		subnetSpec := &b.Cluster.Spec.Subnets[i]
+		allSubnetsSharedInZone[subnetSpec.Zone] = true
+	}
+
 	for i := range b.Cluster.Spec.Subnets {
 		subnetSpec := &b.Cluster.Spec.Subnets[i]
 		sharedSubnet := subnetSpec.ProviderID != ""
 		if !sharedSubnet {
 			allSubnetsShared = false
+			allSubnetsSharedInZone[subnetSpec.Zone] = false
 		}
 	}
 
@@ -124,7 +131,11 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(igw)
 
 		if !allSubnetsShared {
-			routeTableTags := b.CloudTags(vpcName, sharedVPC)
+			// The route table is not shared if we're creating a subnet for our cluster
+			// That subnet will be owned, and will be associated with our RouteTable.
+			// On deletion we delete the subnet & the route table.
+			sharedRouteTable := false
+			routeTableTags := b.CloudTags(vpcName, sharedRouteTable)
 			routeTableTags[awsup.TagNameKopsRole] = "public"
 			publicRouteTable = &awstasks.RouteTable{
 				Name:      s(b.ClusterName()),
@@ -133,7 +144,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				VPC: b.LinkToVPC(),
 
 				Tags:   routeTableTags,
-				Shared: fi.Bool(sharedVPC),
+				Shared: fi.Bool(sharedRouteTable),
 			}
 			c.AddTask(publicRouteTable)
 
@@ -288,14 +299,17 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		// Private Route Table
 		//
 		// The private route table that will route to the NAT Gateway
-		routeTableTags := b.CloudTags(b.NamePrivateRouteTableInZone(zone), sharedVPC)
+		// We create an owned route table if we created any subnet in that zone.
+		// Otherwise we consider it shared.
+		routeTableShared := allSubnetsSharedInZone[zone]
+		routeTableTags := b.CloudTags(b.NamePrivateRouteTableInZone(zone), routeTableShared)
 		routeTableTags[awsup.TagNameKopsRole] = "private-" + zone
 		rt := &awstasks.RouteTable{
 			Name:      s(b.NamePrivateRouteTableInZone(zone)),
 			VPC:       b.LinkToVPC(),
 			Lifecycle: b.Lifecycle,
 
-			Shared: fi.Bool(sharedVPC),
+			Shared: fi.Bool(routeTableShared),
 			Tags:   routeTableTags,
 		}
 		c.AddTask(rt)

pkg/testutils/integrationtestharness.go

@@ -159,6 +159,17 @@ func (h *IntegrationTestHarness) SetupMockAWS() {
 		VpcId:             aws.String("vpc-12345678"),
 	})
 
+	mockEC2.CreateSubnetWithId(&ec2.CreateSubnetInput{
+		VpcId:            aws.String("vpc-12345678"),
+		AvailabilityZone: aws.String("us-test-1a"),
+		CidrBlock:        aws.String("172.20.32.0/19"),
+	}, "subnet-12345678")
+	mockEC2.CreateSubnetWithId(&ec2.CreateSubnetInput{
+		VpcId:            aws.String("vpc-12345678"),
+		AvailabilityZone: aws.String("us-test-1a"),
+		CidrBlock:        aws.String("172.20.4.0/22"),
+	}, "subnet-abcdef")
+
 	mockEC2.AllocateAddressWithId(&ec2.AllocateAddressInput{
 		Address: aws.String("123.45.67.8"),
 	}, "eip-12345678")

tests/integration/update_cluster/private-shared-subnet/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/private-shared-subnet/in-v1alpha2.yaml

@@ -0,0 +1,103 @@
+apiVersion: kops/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-12T04:13:14Z"
+  name: private-shared-subnet.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/private-shared-subnet.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+  kubernetesVersion: v1.8.2
+  masterInternalName: api.internal.private-shared-subnet.example.com
+  masterPublicName: api.private-shared-subnet.example.com
+  networkCIDR: 172.20.0.0/16
+  networkID: vpc-12345678
+  networking:
+    weave: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+  - 0.0.0.0/0
+  topology:
+    masters: private
+    nodes: private
+  subnets:
+  - cidr: 172.20.32.0/19
+    id: subnet-12345678
+    name: us-test-1a
+    type: Private
+    egress: nat-12345678
+    zone: us-test-1a
+  - cidr: 172.20.4.0/22
+    id: subnet-abcdef
+    name: utility-us-test-1a
+    type: Utility
+    zone: us-test-1a
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-12T04:13:15Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: private-shared-subnet.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-12T04:13:15Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: private-shared-subnet.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-14T15:32:41Z"
+  name: bastion
+  labels:
+    kops.k8s.io/cluster: private-shared-subnet.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09
+  machineType: t2.micro
+  maxSize: 1
+  minSize: 1
+  role: Bastion
+  subnets:
+  - utility-us-test-1a